diff --git a/plugins/nf-nomad/src/main/nextflow/nomad/models/ConstraintsBuilder.groovy b/plugins/nf-nomad/src/main/nextflow/nomad/models/ConstraintsBuilder.groovy index 3659f31..3275e19 100644 --- a/plugins/nf-nomad/src/main/nextflow/nomad/models/ConstraintsBuilder.groovy +++ b/plugins/nf-nomad/src/main/nextflow/nomad/models/ConstraintsBuilder.groovy @@ -25,72 +25,22 @@ class ConstraintsBuilder { } protected static List nodeConstraints(JobConstraintsNode nodeSpec){ - def ret = [] as List - if( nodeSpec.id ){ - ret.add new Constraint() - .ltarget('${node.unique.id}') - .operand("=") - .rtarget(nodeSpec.id) - } - if( nodeSpec.name ){ - ret.add new Constraint() - .ltarget('${node.unique.name}') - .operand("=") - .rtarget(nodeSpec.name) - } - if( nodeSpec.clientClass ){ - ret.add new Constraint() - .ltarget('${node.class}') - .operand("=") - .rtarget(nodeSpec.clientClass) - } - if( nodeSpec.dataCenter ){ - ret.add new Constraint() - .ltarget('${node.datacenter}') - .operand("=") - .rtarget(nodeSpec.dataCenter) - } - if( nodeSpec.region ){ - ret.add new Constraint() - .ltarget('${node.region}') - .operand("=") - .rtarget(nodeSpec.region) - } - if( nodeSpec.pool ){ - ret.add new Constraint() - .ltarget('${node.pool}') - .operand("=") - .rtarget(nodeSpec.pool) - } + def ret = nodeSpec.raws?.collect{ triple-> + return new Constraint() + .ltarget('${'+triple.left+'}') + .operand(triple.middle) + .rtarget(triple.right) + } as List ret } protected static List attrConstraints(JobConstraintsAttr nodeSpec) { - def ret = [] as List - if (nodeSpec.arch) { - ret.add new Constraint() - .ltarget('${attr.cpu.arch}') - .operand("=") - .rtarget(nodeSpec.arch) - } - if (nodeSpec.numcores) { - ret.add new Constraint() - .ltarget('${attr.cpu.numcores}') - .operand("=") - .rtarget("$nodeSpec.numcores") - } - if (nodeSpec.reservablecores) { - ret.add new Constraint() - .ltarget('${attr.cpu.reservablecores}') - .operand("=") - .rtarget("$nodeSpec.reservablecores") - } - if (nodeSpec.totalcompute) { - ret.add new Constraint() - .ltarget('${attr.cpu.totalcompute}') - .operand("=") - .rtarget("$nodeSpec.totalcompute") - } + def ret = nodeSpec.raws?.collect{ triple-> + return new Constraint() + .ltarget('${'+triple.left+'}') + .operand(triple.middle) + .rtarget(triple.right) + } as List ret } diff --git a/plugins/nf-nomad/src/main/nextflow/nomad/models/JobConstraintsAttr.groovy b/plugins/nf-nomad/src/main/nextflow/nomad/models/JobConstraintsAttr.groovy index 5b0c1eb..875b158 100644 --- a/plugins/nf-nomad/src/main/nextflow/nomad/models/JobConstraintsAttr.groovy +++ b/plugins/nf-nomad/src/main/nextflow/nomad/models/JobConstraintsAttr.groovy @@ -18,6 +18,8 @@ package nextflow.nomad.models +import org.apache.commons.lang3.tuple.Triple + /** * Nomad Job Constraint Spec * @@ -26,37 +28,56 @@ package nextflow.nomad.models class JobConstraintsAttr { - private String arch = null - private Integer numcores= null - private Integer reservablecores= null - private Double totalcompute= null + private List> raws= [] - String getArch() { - return arch + List> getRaws() { + return raws } - Integer getNumcores() { - return numcores + JobConstraintsAttr setCpu(Map map){ + cpu(map) } - Integer getReservablecores() { - return reservablecores + JobConstraintsAttr cpu(Map map){ + if( map.containsKey('arch')) + raw("cpu.arch","=", map['arch'].toString()) + if( map.containsKey('numcores')) + raw("cpu.numcores",">=", map['numcores'].toString()) + if( map.containsKey('reservablecores')) + raw("cpu.reservablecores",">=", map['reservablecores'].toString()) + if( map.containsKey('totalcompute')) + raw("cpu.totalcompute","=", map['totalcompute'].toString()) + this } - Double getTotalcompute() { - return totalcompute + JobConstraintsAttr setUnique(Map map){ + unique(map) } - JobConstraintsAttr setCpu(Map map){ - cpu(map) + JobConstraintsAttr unique(Map map){ + if( map.containsKey('hostname')) + raw("unique.hostname","=", map['hostname'].toString()) + if( map.containsKey('ip-address')) + raw("unique.network.ip-address","=", map['ip-address'].toString()) + this } - JobConstraintsAttr cpu(Map map){ - this.arch = map.containsKey("arch") ? map["arch"].toString() : null - this.numcores = map.containsKey("numcores") ? map["numcores"] as int : null - this.reservablecores = map.containsKey("reservablecores") ? map["reservablecores"] as int : null - this.totalcompute = map.containsKey("totalcompute") ? map["totalcompute"] as double : null + JobConstraintsAttr setKernel(Map map){ + kernel(map) + } + + JobConstraintsAttr kernel(Map map){ + if( map.containsKey('arch')) + raw("kernel.arch","=", map['arch'].toString()) + if( map.containsKey('name')) + raw("kernel.name","=", map['name'].toString()) + if( map.containsKey('version')) + raw("kernel.version","=", map['version'].toString()) this } + JobConstraintsAttr raw(String attr, String operator, String value){ + raws.add Triple.of("attr."+attr, operator, value) + this + } } diff --git a/plugins/nf-nomad/src/main/nextflow/nomad/models/JobConstraintsNode.groovy b/plugins/nf-nomad/src/main/nextflow/nomad/models/JobConstraintsNode.groovy index f02f4fa..3ac0f9e 100644 --- a/plugins/nf-nomad/src/main/nextflow/nomad/models/JobConstraintsNode.groovy +++ b/plugins/nf-nomad/src/main/nextflow/nomad/models/JobConstraintsNode.groovy @@ -18,6 +18,8 @@ package nextflow.nomad.models +import org.apache.commons.lang3.tuple.Triple + /** * Nomad Job Constraint Spec * @@ -26,35 +28,10 @@ package nextflow.nomad.models class JobConstraintsNode { - private String id = null - private String name = null - private String clientClass = null - private String pool = null - private String dataCenter = null - private String region = null - - String getId() { - return id - } - - String getName() { - return name - } - - String getClientClass() { - return clientClass - } - - String getPool() { - return pool - } - - String getDataCenter() { - return dataCenter - } + private List> raws= [] - String getRegion() { - return region + List> getRaws() { + return raws } JobConstraintsNode setUnique(Map map){ @@ -62,17 +39,19 @@ class JobConstraintsNode { } JobConstraintsNode unique(Map map){ - this.id = map.containsKey("id") ? map["id"].toString() : null - this.name = map.containsKey("name") ? map["name"].toString() : null + ['id', 'name'].each { key-> + if( map.containsKey(key)) + raw("unique.${key}","=", map[key].toString()) + } this } - JobConstraintsNode setClientClass(Object map){ - clientClass(map) + JobConstraintsNode setClazz(Object map){ // class is a reserved word, in java we used clazz + clazz(map) } - JobConstraintsNode clientClass(Object clientClass){ - this.clientClass = clientClass.toString() + JobConstraintsNode clazz(Object cls){ + raw("class","=", cls.toString()) this } @@ -81,7 +60,7 @@ class JobConstraintsNode { } JobConstraintsNode pool(Object pool){ - this.pool = pool.toString() + raw("pool","=", pool.toString()) this } @@ -90,7 +69,7 @@ class JobConstraintsNode { } JobConstraintsNode dataCenter(Object dataCenter){ - this.dataCenter = dataCenter.toString() + raw("datacenter","=", dataCenter.toString()) this } @@ -99,7 +78,12 @@ class JobConstraintsNode { } JobConstraintsNode region(Object region){ - this.region = region.toString() + raw("region","=", region.toString()) + this + } + + JobConstraintsNode raw(String attr, String operator, String value){ + raws.add Triple.of("node."+attr, operator, value) this } } diff --git a/plugins/nf-nomad/src/test/nextflow/nomad/config/NomadJobConstraints.groovy b/plugins/nf-nomad/src/test/nextflow/nomad/config/NomadJobConstraints.groovy index d4b9692..f67006c 100644 --- a/plugins/nf-nomad/src/test/nextflow/nomad/config/NomadJobConstraints.groovy +++ b/plugins/nf-nomad/src/test/nextflow/nomad/config/NomadJobConstraints.groovy @@ -35,7 +35,7 @@ class NomadJobConstraints extends Specification { constraints: { node { unique = [id :"node-id", name: "node-name"] - clientClass = "linux-64bit" + clazz = "linux-64bit" pool = "custom-pool" dataCenter = 'dc1' region = 'us' @@ -50,15 +50,15 @@ class NomadJobConstraints extends Specification { then: config.jobOpts.constraintsSpec config.jobOpts.constraintsSpec.nodeSpecs.size() == 1 - config.jobOpts.constraintsSpec.nodeSpecs[0].id == "node-id" - config.jobOpts.constraintsSpec.nodeSpecs[0].name == "node-name" - config.jobOpts.constraintsSpec.nodeSpecs[0].clientClass == "linux-64bit" - config.jobOpts.constraintsSpec.nodeSpecs[0].pool == "custom-pool" - config.jobOpts.constraintsSpec.nodeSpecs[0].dataCenter == "dc1" - config.jobOpts.constraintsSpec.nodeSpecs[0].region == "us" + config.jobOpts.constraintsSpec.nodeSpecs[0].raws.find{it.left.endsWith("id")}.right == "node-id" + config.jobOpts.constraintsSpec.nodeSpecs[0].raws.find{it.left.endsWith("name")}.right == "node-name" + config.jobOpts.constraintsSpec.nodeSpecs[0].raws.find{it.left.endsWith("class")}.right == "linux-64bit" + config.jobOpts.constraintsSpec.nodeSpecs[0].raws.find{it.left.endsWith("pool")}.right == "custom-pool" + config.jobOpts.constraintsSpec.nodeSpecs[0].raws.find{it.left.endsWith("datacenter")}.right == "dc1" + config.jobOpts.constraintsSpec.nodeSpecs[0].raws.find{it.left.endsWith("region")}.right == "us" config.jobOpts.constraintsSpec.attrSpecs.size() == 1 - config.jobOpts.constraintsSpec.attrSpecs[0].arch == '286' + config.jobOpts.constraintsSpec.attrSpecs[0].raws[0].right == '286' } void "should instantiate a no completed constraints spec"() { @@ -68,7 +68,7 @@ class NomadJobConstraints extends Specification { constraints: { node { unique = [id :"node-id", name: "node-name"] - clientClass = "linux-64bit" + clazz = "linux-64bit" } } ] @@ -77,11 +77,11 @@ class NomadJobConstraints extends Specification { then: config.jobOpts.constraintsSpec config.jobOpts.constraintsSpec.nodeSpecs.size() - config.jobOpts.constraintsSpec.nodeSpecs[0].id == "node-id" - config.jobOpts.constraintsSpec.nodeSpecs[0].name == "node-name" - config.jobOpts.constraintsSpec.nodeSpecs[0].clientClass == "linux-64bit" - !config.jobOpts.constraintsSpec.nodeSpecs[0].pool - !config.jobOpts.constraintsSpec.nodeSpecs[0].dataCenter + config.jobOpts.constraintsSpec.nodeSpecs[0].raws.find{it.left.endsWith("id")}.right == "node-id" + config.jobOpts.constraintsSpec.nodeSpecs[0].raws.find{it.left.endsWith("name")}.right == "node-name" + config.jobOpts.constraintsSpec.nodeSpecs[0].raws.find{it.left.endsWith("class")}.right == "linux-64bit" + !config.jobOpts.constraintsSpec.nodeSpecs[0].raws.find{it.left.endsWith("pool")} + !config.jobOpts.constraintsSpec.nodeSpecs[0].raws.find{it.left.endsWith("datacenter")} } void "should instantiate a list of constraints spec if specified"() { @@ -91,14 +91,14 @@ class NomadJobConstraints extends Specification { constraints: { node { unique = [id :"node-id", name: "node-name"] - clientClass = "linux-64bit" + clazz = "linux-64bit" pool = "custom-pool" dataCenter = 'dc1' region = 'us' } node { unique = [id :"node-id", name: "node-name"] - clientClass = "linux-64bit" + clazz = "linux-64bit" pool = "custom-pool" dataCenter = 'dc1' region = 'us' @@ -110,11 +110,11 @@ class NomadJobConstraints extends Specification { then: config.jobOpts.constraintsSpec config.jobOpts.constraintsSpec.nodeSpecs.size() - config.jobOpts.constraintsSpec.nodeSpecs[0].id == "node-id" - config.jobOpts.constraintsSpec.nodeSpecs[0].name == "node-name" - config.jobOpts.constraintsSpec.nodeSpecs[0].clientClass == "linux-64bit" - config.jobOpts.constraintsSpec.nodeSpecs[0].pool == "custom-pool" - config.jobOpts.constraintsSpec.nodeSpecs[0].dataCenter == "dc1" - config.jobOpts.constraintsSpec.nodeSpecs[0].region == "us" + config.jobOpts.constraintsSpec.nodeSpecs[0].raws.find{it.left.endsWith("id")}.right == "node-id" + config.jobOpts.constraintsSpec.nodeSpecs[0].raws.find{it.left.endsWith("name")}.right == "node-name" + config.jobOpts.constraintsSpec.nodeSpecs[0].raws.find{it.left.endsWith("class")}.right == "linux-64bit" + config.jobOpts.constraintsSpec.nodeSpecs[0].raws.find{it.left.endsWith("pool")}.right == "custom-pool" + config.jobOpts.constraintsSpec.nodeSpecs[0].raws.find{it.left.endsWith("datacenter")}.right == "dc1" + config.jobOpts.constraintsSpec.nodeSpecs[0].raws.find{it.left.endsWith("region")}.right == "us" } } diff --git a/plugins/nf-nomad/src/test/nextflow/nomad/models/JobConstraintsSpec.groovy b/plugins/nf-nomad/src/test/nextflow/nomad/models/JobConstraintsSpec.groovy index 1e57c6c..0fc3ae6 100644 --- a/plugins/nf-nomad/src/test/nextflow/nomad/models/JobConstraintsSpec.groovy +++ b/plugins/nf-nomad/src/test/nextflow/nomad/models/JobConstraintsSpec.groovy @@ -250,4 +250,71 @@ class JobConstraintsSpec extends Specification{ body.Job.TaskGroups[0].Tasks[0].Constraints[0].RTarget == '286' body.Job.TaskGroups[0].Tasks[0].Constraints[0].Operand == '=' } + + void "submit a task with a raw attr constraint"(){ + given: + def config = new NomadConfig( + client:[ + address : "http://${mockWebServer.hostName}:${mockWebServer.port}" + ], + ) + def service = new NomadService(config) + + String id = "theId" + String name = "theName" + String image = "theImage" + List args = ["theCommand", "theArgs"] + String workingDir = "/a/b/c" + Mapenv = [test:"test"] + + def contraints = { + attr { + raw 'platform.aws.instance-type', '=', 'm4.xlarge' + } + } + + def mockTask = Mock(TaskRun){ + getName() >> name + getContainer() >> image + getConfig() >> Mock(TaskConfig) + getWorkDirStr() >> workingDir + getContainer() >> "ubuntu" + getProcessor() >> Mock(TaskProcessor){ + getExecutor() >> Mock(Executor){ + isFusionEnabled() >> false + } + getConfig() >> Mock(ProcessConfig){ + get("constraints") >> contraints + } + } + getWorkDir() >> Path.of(workingDir) + toTaskBean() >> Mock(TaskBean){ + getWorkDir() >> Path.of(workingDir) + getScript() >> "theScript" + getShell() >> ["bash"] + getInputFiles() >> [:] + } + } + + mockWebServer.enqueue(new MockResponse() + .setBody(JsonOutput.toJson(["EvalID":"test"]).toString()) + .addHeader("Content-Type", "application/json")); + when: + + def idJob = service.submitTask(id, mockTask, args, env) + def recordedRequest = mockWebServer.takeRequest(); + def body = new JsonSlurper().parseText(recordedRequest.body.readUtf8()) + + then: + idJob + + and: + recordedRequest.method == "POST" + recordedRequest.path == "/v1/jobs" + + and: + body.Job.TaskGroups[0].Tasks[0].Constraints[0].LTarget == '${attr.platform.aws.instance-type}' + body.Job.TaskGroups[0].Tasks[0].Constraints[0].RTarget == 'm4.xlarge' + body.Job.TaskGroups[0].Tasks[0].Constraints[0].Operand == '=' + } }