diff --git a/buildSrc/src/main/groovy/nextflow/gradle/plugins/GenerateIdxTask.groovy b/buildSrc/src/main/groovy/nextflow/gradle/plugins/GenerateIdxTask.groovy index 4b8708f..289f475 100644 --- a/buildSrc/src/main/groovy/nextflow/gradle/plugins/GenerateIdxTask.groovy +++ b/buildSrc/src/main/groovy/nextflow/gradle/plugins/GenerateIdxTask.groovy @@ -34,6 +34,7 @@ abstract class GenerateIdxTask extends DefaultTask{ def matcher = new SourcesMatcher(project) def extensionsClassName = matcher.pluginExtensions + extensionsClassName += matcher.providers def traceClassName = matcher.traceObservers def all = extensionsClassName+traceClassName output.text = all.join('\n') diff --git a/buildSrc/src/main/groovy/nextflow/gradle/plugins/SourcesMatcher.groovy b/buildSrc/src/main/groovy/nextflow/gradle/plugins/SourcesMatcher.groovy index 1865b93..84237bd 100644 --- a/buildSrc/src/main/groovy/nextflow/gradle/plugins/SourcesMatcher.groovy +++ b/buildSrc/src/main/groovy/nextflow/gradle/plugins/SourcesMatcher.groovy @@ -21,6 +21,11 @@ class SourcesMatcher { findSources(/class (\w+) extends Executor implements ExtensionPoint/) } + List getProviders(){ + return findSources(/class (\w+) implements (.+)Provider/) + } + + List getTraceObservers(){ return findSources(/class (\w+) implements TraceObserverFactory/) } diff --git a/plugins/nf-nomad/src/main/nextflow/nomad/NomadPlugin.groovy b/plugins/nf-nomad/src/main/nextflow/nomad/NomadPlugin.groovy index 5772093..5273ae8 100644 --- a/plugins/nf-nomad/src/main/nextflow/nomad/NomadPlugin.groovy +++ b/plugins/nf-nomad/src/main/nextflow/nomad/NomadPlugin.groovy @@ -21,6 +21,7 @@ import groovy.transform.CompileStatic import nextflow.nomad.executor.TaskDirectives import nextflow.plugin.BasePlugin import nextflow.script.ProcessConfig +import nextflow.secret.SecretsLoader import org.pf4j.PluginWrapper /** @@ -35,6 +36,7 @@ class NomadPlugin extends BasePlugin { NomadPlugin(PluginWrapper wrapper) { super(wrapper) addCustomDirectives() + SecretsLoader.instance.reset() } private static void addCustomDirectives() { diff --git a/plugins/nf-nomad/src/main/nextflow/nomad/NomadSecretProvider.groovy b/plugins/nf-nomad/src/main/nextflow/nomad/NomadSecretProvider.groovy new file mode 100644 index 0000000..668b846 --- /dev/null +++ b/plugins/nf-nomad/src/main/nextflow/nomad/NomadSecretProvider.groovy @@ -0,0 +1,60 @@ +package nextflow.nomad + +import groovy.util.logging.Slf4j +import nextflow.plugin.Priority +import nextflow.secret.Secret +import nextflow.secret.SecretsProvider + +@Slf4j +@Priority(-100) // high priority +class NomadSecretProvider implements SecretsProvider, Closeable{ + + @Override + void close() throws IOException { + } + + @Override + boolean activable() { + return true + } + + @Override + SecretsProvider load() { + this + } + + @Override + Secret getSecret(String name) { + log.error("NomadSecretProvider can't get secret, use nomad cli or disable it") + null + } + + @Override + String getSecretsEnv(List secretNames) { + log.error("NomadSecretProvider can't get secret, use nomad cli or disable it") + null + } + + @Override + String getSecretsEnv() { + log.error("NomadSecretProvider can't get secret, use nomad cli or disable it") + null + } + + @Override + void putSecret(String name, String value) { + throw new UnsupportedOperationException("NomadSecretProvider can't put secret, use nomad cli") + } + + @Override + void removeSecret(String name) { + throw new UnsupportedOperationException("NomadSecretProvider can't remove secret, use nomad cli") + } + + @Override + Set listSecretsNames() { + log.error("NomadSecretProvider can't get secret, use nomad cli or disable it") + null + } + +} diff --git a/plugins/nf-nomad/src/main/nextflow/nomad/executor/NomadService.groovy b/plugins/nf-nomad/src/main/nextflow/nomad/executor/NomadService.groovy index 21cc80c..5325304 100644 --- a/plugins/nf-nomad/src/main/nextflow/nomad/executor/NomadService.groovy +++ b/plugins/nf-nomad/src/main/nextflow/nomad/executor/NomadService.groovy @@ -20,6 +20,7 @@ package nextflow.nomad.executor import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.nomadproject.client.ApiClient +import io.nomadproject.client.ApiException import io.nomadproject.client.api.JobsApi import io.nomadproject.client.model.* import nextflow.nomad.models.ConstraintsBuilder @@ -110,6 +111,9 @@ class NomadService implements Closeable{ try { JobRegisterResponse jobRegisterResponse = jobsApi.registerJob(jobRegisterRequest, config.jobOpts().region, config.jobOpts().namespace, null, null) jobRegisterResponse.evalID + } catch( ApiException apiException){ + log.debug("[NOMAD] Failed to submit ${job.name} -- Cause: ${apiException.responseBody ?: apiException}", apiException) + throw new ProcessSubmitException("[NOMAD] Failed to submit ${job.name} -- Cause: ${apiException.responseBody ?: apiException}", apiException) } catch (Throwable e) { log.debug("[NOMAD] Failed to submit ${job.name} -- Cause: ${e.message ?: e}", e) throw new ProcessSubmitException("[NOMAD] Failed to submit ${job.name} -- Cause: ${e.message ?: e}", e) @@ -186,7 +190,7 @@ class NomadService implements Closeable{ affinity(task, taskDef) constraint(task, taskDef) constraints(task, taskDef) - + secrets(task, taskDef) return taskDef } @@ -276,7 +280,18 @@ class NomadService implements Closeable{ taskDef } - + protected Task secrets(TaskRun task, Task taskDef){ + def secrets = task.processor?.config?.get(TaskDirectives.SECRETS) + if( secrets ){ + Template template = new Template(envvars: true, destPath: "/secrets/nf-nomad") + String tmpl = secrets.collect{ String name-> + "${name}={{ with nomadVar \"secrets/${name}\" }}{{ .${name} }}{{ end }}" + }.join('\n').stripIndent() + template.embeddedTmpl(tmpl) + taskDef.addTemplatesItem(template) + } + taskDef + } protected Job assignDatacenters(TaskRun task, Job job){ def datacenters = task.processor?.config?.get(TaskDirectives.DATACENTERS) diff --git a/plugins/nf-nomad/src/main/nextflow/nomad/executor/TaskDirectives.groovy b/plugins/nf-nomad/src/main/nextflow/nomad/executor/TaskDirectives.groovy index edc1722..e72c381 100644 --- a/plugins/nf-nomad/src/main/nextflow/nomad/executor/TaskDirectives.groovy +++ b/plugins/nf-nomad/src/main/nextflow/nomad/executor/TaskDirectives.groovy @@ -6,8 +6,11 @@ class TaskDirectives { public static final String CONSTRAINTS = "constraints" + public static final String SECRETS = "secret" + public static final List ALL = [ DATACENTERS, - CONSTRAINTS + CONSTRAINTS, + SECRETS ] } diff --git a/validation/secrets/main.nf b/validation/secrets/main.nf new file mode 100644 index 0000000..3304707 --- /dev/null +++ b/validation/secrets/main.nf @@ -0,0 +1,20 @@ +#!/usr/bin/env nextflow + +process sayHello { + container 'ubuntu:20.04' + secret 'MY_ACCESS_KEY' + secret 'MY_SECRET_KEY' + + input: + val x + output: + stdout + + """ + echo $x world! the access \$MY_ACCESS_KEY and the secret \$MY_SECRET_KEY + """ +} + +workflow { + Channel.of('Bonjour', 'Ciao', 'Hello', 'Hola') | sayHello | view +} diff --git a/validation/secrets/nextflow.config b/validation/secrets/nextflow.config new file mode 100644 index 0000000..d35b509 --- /dev/null +++ b/validation/secrets/nextflow.config @@ -0,0 +1,21 @@ +plugins { + id "nf-nomad@${System.getenv("NOMAD_PLUGIN_VERSION") ?: "latest"}" +} + +process { + executor = "nomad" +} + +nomad { + + client { + address = "http://localhost:4646" + } + + jobs { + deleteOnCompletion = false + volume = { type "host" name "scratchdir" } + } + +} +