diff --git a/Makefile b/Makefile index 0474825c1..5b6206936 100644 --- a/Makefile +++ b/Makefile @@ -45,7 +45,7 @@ test: test-integration: @echo "==> Testing ${NAME} (test suite & integration)" @go test -count=1 -timeout=80s -tags=integration -cover ./... ${TESTARGS} -.PHONY: test-all +.PHONY: test-integration # test-e2e runs e2e tests test-e2e: dev diff --git a/client/terraform_cli_test.go b/client/terraform_cli_test.go index 3f1ab6119..752856456 100644 --- a/client/terraform_cli_test.go +++ b/client/terraform_cli_test.go @@ -350,7 +350,7 @@ module for task "test-workspace" is missing the "catalog_services" variable, add } } ] -}`, // Terraform v0.13/v0.14 output +}`, // Terraform v0.13/v0.14 output }, { "warning", diff --git a/config/condition.go b/config/condition.go index 647beb34e..e84aad49e 100644 --- a/config/condition.go +++ b/config/condition.go @@ -41,6 +41,8 @@ func isConditionNil(c ConditionConfig) bool { result = v == nil case *CatalogServicesConditionConfig: result = v == nil + case *NodesConditionConfig: + result = v == nil default: return c == nil || reflect.ValueOf(c).IsNil() } @@ -84,6 +86,10 @@ func conditionToTypeFunc() mapstructure.DecodeHookFunc { var config ServicesConditionConfig return decodeConditionToType(c, &config) } + if c, ok := conditions[nodesConditionTypes]; ok { + var config NodesConditionConfig + return decodeConditionToType(c, &config) + } return nil, fmt.Errorf("unsupported condition type: %v", data) } diff --git a/config/condition_nodes.go b/config/condition_nodes.go new file mode 100644 index 000000000..32e9c69d5 --- /dev/null +++ b/config/condition_nodes.go @@ -0,0 +1,88 @@ +package config + +import "fmt" + +const nodesConditionTypes = "nodes" + +type NodesConditionConfig struct { + Datacenter *string `mapstructure:"datacenter"` + + services []string +} + +func (n *NodesConditionConfig) Copy() ConditionConfig { + if n == nil { + return nil + } + + var o NodesConditionConfig + o.Datacenter = n.Datacenter + copy(o.services, n.services) + + return &o +} + +func (n *NodesConditionConfig) Merge(o ConditionConfig) ConditionConfig { + if n == nil { + if isConditionNil(o) { + return nil + } + return o.Copy() + } + + if isConditionNil(o) { + return n.Copy() + } + + r := n.Copy() + o2, ok := o.(*NodesConditionConfig) + if !ok { + return r + } + + r2 := r.(*NodesConditionConfig) + + if o2.Datacenter != nil { + r2.Datacenter = StringCopy(o2.Datacenter) + } + + r2.services = append(r2.services, o2.services...) + + return r2 +} + +func (n *NodesConditionConfig) Finalize(services []string) { + if n == nil { + return + } + + if n.Datacenter == nil { + n.Datacenter = String("") + } + + n.services = services +} + +func (n *NodesConditionConfig) Validate() error { + if n == nil { + return nil + } + + if len(n.services) != 0 { + return fmt.Errorf("task.services cannot be set when using condition %q", nodesConditionTypes) + } + + return nil +} + +func (n *NodesConditionConfig) GoString() string { + if n == nil { + return "(*NodesConditionConfig)(nil)" + } + + return fmt.Sprintf("&NodesConditionConfig{"+ + "Datacenter:%s, "+ + "}", + StringVal(n.Datacenter), + ) +} diff --git a/config/condition_nodes_test.go b/config/condition_nodes_test.go new file mode 100644 index 000000000..de236c5e2 --- /dev/null +++ b/config/condition_nodes_test.go @@ -0,0 +1,165 @@ +package config + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNodesConditionConfig_Copy(t *testing.T) { + cases := []struct { + name string + a *NodesConditionConfig + }{ + { + "nil", + nil, + }, + { + "empty", + &NodesConditionConfig{}, + }, + { + "fully_configured", + &NodesConditionConfig{ + Datacenter: String("dc2"), + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + r := tc.a.Copy() + if tc.a == nil { + // returned nil interface has nil type, which is unequal to tc.a + assert.Nil(t, r) + } else { + assert.Equal(t, tc.a, r) + } + }) + } +} + +func TestNodesConditionConfig_Merge(t *testing.T) { + cases := []struct { + name string + a *NodesConditionConfig + b *NodesConditionConfig + r *NodesConditionConfig + }{ + { + "nil_a", + nil, + &NodesConditionConfig{}, + &NodesConditionConfig{}, + }, + { + "nil_b", + &NodesConditionConfig{}, + nil, + &NodesConditionConfig{}, + }, + { + "nil_both", + nil, + nil, + nil, + }, + { + "empty", + &NodesConditionConfig{}, + &NodesConditionConfig{}, + &NodesConditionConfig{}, + }, + { + "datacenter_overrides", + &NodesConditionConfig{Datacenter: String("same")}, + &NodesConditionConfig{Datacenter: String("different")}, + &NodesConditionConfig{Datacenter: String("different")}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + r := tc.a.Merge(tc.b) + if tc.r == nil { + // returned nil interface has nil type, which is unequal to tc.r + assert.Nil(t, r) + } else { + assert.Equal(t, tc.r, r) + } + }) + } +} + +func TestNodesConditionConfig_Finalize(t *testing.T) { + cases := []struct { + name string + s []string + i *NodesConditionConfig + r *NodesConditionConfig + }{ + { + "empty", + []string{}, + &NodesConditionConfig{}, + &NodesConditionConfig{ + Datacenter: String(""), + services: []string{}, + }, + }, + { + "pass_in_services", + []string{"api"}, + &NodesConditionConfig{}, + &NodesConditionConfig{ + Datacenter: String(""), + services: []string{"api"}, + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + tc.i.Finalize(tc.s) + assert.Equal(t, tc.r, tc.i) + }) + } +} + +func TestNodesConditionConfig_Validate(t *testing.T) { + cases := []struct { + name string + expectErr bool + services []string + }{ + { + "nil", + false, + nil, + }, + { + "empty", + false, + []string{}, + }, + { + "services set", + true, + []string{"api"}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + n := &NodesConditionConfig{} + n.Finalize(tc.services) + err := n.Validate() + if tc.expectErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} diff --git a/config/condition_test.go b/config/condition_test.go index 7836770f2..8c61b30e7 100644 --- a/config/condition_test.go +++ b/config/condition_test.go @@ -141,6 +141,58 @@ task { nonexistent_field = true } }`, + }, + { + "nodes: empty", + false, + &NodesConditionConfig{ + Datacenter: String(""), + services: []string{}, + }, + "config.hcl", + ` +task { + name = "condition_task" + source = "..." + services = [] + condition "nodes" {} +} +`, + }, + { + "nodes: happy path", + false, + &NodesConditionConfig{ + Datacenter: String("dc2"), + services: []string{}, + }, + "config.hcl", + ` +task { + name = "condition_task" + source = "..." + services = [] + condition "nodes" { + datacenter = "dc2" + } +} +`, + }, + { + "nodes: unsupported field", + true, + nil, + "config.hcl", + ` +task { + name = "condition_task" + source = "..." + services = [] + condition "nodes" { + nonexistent_field = true + } +} +`, }, { "error: two conditions", diff --git a/driver/task.go b/driver/task.go index 43c2b3953..7b13e922d 100644 --- a/driver/task.go +++ b/driver/task.go @@ -320,6 +320,10 @@ func (t *Task) configureRootModuleInput(input *tftmpl.RootModuleInputData) { condition = &tftmpl.ServicesCondition{ Regexp: *v.Regexp, } + case *config.NodesConditionConfig: + condition = &tftmpl.NodesCondition{ + Datacenter: *v.Datacenter, + } default: // expected only for test scenarios log.Printf("[WARN] (driver.terraform) task '%s' condition config unset."+ diff --git a/templates/tftmpl/condition.go b/templates/tftmpl/condition.go index cbbd21ae4..a690c6b16 100644 --- a/templates/tftmpl/condition.go +++ b/templates/tftmpl/condition.go @@ -202,3 +202,83 @@ variable "catalog_services" { type = map(list(string)) } `) + +type NodesCondition struct { + Datacenter string + Filter string +} + +func (n NodesCondition) SourceIncludesVariable() bool { + return true +} + +func (n NodesCondition) ServicesAppended() bool { + return false +} + +func (n NodesCondition) appendModuleAttribute(body *hclwrite.Body) { + body.SetAttributeTraversal("nodes", hcl.Traversal{ + hcl.TraverseRoot{Name: "var"}, + hcl.TraverseAttr{Name: "nodes"}, + }) +} + +func (n NodesCondition) appendTemplate(w io.Writer) error { + q := n.hcatQuery() + _, err := fmt.Fprintf(w, nodesRegexTmpl, q) + if err != nil { + log.Printf("[ERR] (templates.tftmpl) unable to write nodes condition template") + return err + } + + return nil +} + +func (n NodesCondition) appendVariable(w io.Writer) error { + _, err := w.Write(variableNodes) + return err +} + +func (n NodesCondition) hcatQuery() string { + var opts []string + + if n.Datacenter != "" { + opts = append(opts, fmt.Sprintf("datacenter=%s", n.Datacenter)) + } + + if n.Filter != "" { + opts = append(opts, fmt.Sprintf("filter=%s", n.Filter)) + } + + if len(opts) > 0 { + return `"` + strings.Join(opts, `" "`) + `" ` // deliberate space at end + } + return "" +} + +const nodesRegexTmpl = ` +nodes = [ +{{- with $nodes := nodes %s }} + {{- range $node := $nodes}} + { +{{ HCLNode $node | indent 4 }} + }, + {{- end}} +{{- end}} +] +` + +var variableNodes = []byte(` +# Nodes definition protocol v0 +variable "nodes" { + description = "Consul nodes" + type = list(object({ + id = string + node = string + address = string + datacenter = string + tagged_addresses = map(string) + meta = map(string) + })) +} +`) diff --git a/templates/tftmpl/tmplfunc/hcl_service_func.go b/templates/tftmpl/tmplfunc/hcl_service_func.go index 96ed479a6..06e59cf2e 100644 --- a/templates/tftmpl/tmplfunc/hcl_service_func.go +++ b/templates/tftmpl/tmplfunc/hcl_service_func.go @@ -97,3 +97,31 @@ func nonNullMap(m map[string]string) map[string]string { return m } + +func hclNodeFunc(nDep *dep.Node) string { + if nDep == nil { + return "" + } + + n := node{ + ID: nDep.ID, + Node: nDep.Node, + Address: nDep.Address, + Datacenter: nDep.Datacenter, + TaggedAddresses: nDep.TaggedAddresses, + Meta: nDep.Meta, + } + + f := hclwrite.NewEmptyFile() + gohcl.EncodeIntoBody(n, f.Body()) + return strings.TrimSpace(string(f.Bytes())) +} + +type node struct { + ID string `hcl:"id"` + Node string `hcl:"node"` + Address string `hcl:"address"` + Datacenter string `hcl:"datacenter"` + TaggedAddresses map[string]string `hcl:"tagged_addresses"` + Meta map[string]string `hcl:"meta"` +} diff --git a/templates/tftmpl/tmplfunc/nodes.go b/templates/tftmpl/tmplfunc/nodes.go new file mode 100644 index 000000000..bf7293d20 --- /dev/null +++ b/templates/tftmpl/tmplfunc/nodes.go @@ -0,0 +1,140 @@ +package tmplfunc + +import ( + "fmt" + "sort" + "strings" + + consulapi "github.com/hashicorp/consul/api" + "github.com/hashicorp/go-bexpr" + "github.com/hashicorp/hcat" + "github.com/hashicorp/hcat/dep" + "github.com/pkg/errors" +) + +func nodesFunc(recall hcat.Recaller) interface{} { + return func(opts ...string) ([]*dep.Node, error) { + result := []*dep.Node{} + + d, err := newNodesQuery(opts) + if err != nil { + return nil, err + } + + if value, ok := recall(d); ok { + return value.([]*dep.Node), nil + } + + return result, nil + } +} + +type nodesQuery struct { + stopCh chan struct{} + + dc string + filter string + opts consulapi.QueryOptions +} + +func newNodesQuery(opts []string) (*nodesQuery, error) { + nodesQuery := nodesQuery{ + stopCh: make(chan struct{}, 1), + } + var filters []string + for _, opt := range opts { + if strings.TrimSpace(opt) == "" { + continue + } + + if queryParamOptRe.MatchString(opt) { + queryParam := strings.SplitN(opt, "=", 2) + query := strings.TrimSpace(queryParam[0]) + value := strings.TrimSpace(queryParam[1]) + switch query { + case "dc", "datacenter": + nodesQuery.dc = value + continue + } + } + + _, err := bexpr.CreateFilter(opt) + if err != nil { + return nil, fmt.Errorf("nodes: invalid filter: %q: %s", opt, err) + } + filters = append(filters, opt) + } + + if len(filters) > 0 { + nodesQuery.filter = strings.Join(filters, " and ") + } + + return &nodesQuery, nil +} + +func (d *nodesQuery) Fetch(clients dep.Clients) (interface{}, *dep.ResponseMetadata, error) { + select { + case <-d.stopCh: + return nil, nil, dep.ErrStopped + default: + } + + opts := d.opts + if d.dc != "" { + opts.Datacenter = d.dc + } + if d.filter != "" { + opts.Filter = d.filter + } + catalog, qm, err := clients.Consul().Catalog().Nodes(&opts) + if err != nil { + return nil, nil, errors.Wrap(err, d.String()) + } + + var nodes []*dep.Node + for _, n := range catalog { + nodes = append(nodes, &dep.Node{ + ID: n.ID, + Node: n.Node, + Address: n.Address, + Datacenter: n.Datacenter, + TaggedAddresses: n.TaggedAddresses, + Meta: n.Meta, + }) + } + + sort.Stable(ByID(nodes)) + + rm := &dep.ResponseMetadata{ + LastIndex: qm.LastIndex, + LastContact: qm.LastContact, + } + d.opts.WaitIndex = qm.LastIndex + + return nodes, rm, nil +} + +func (d *nodesQuery) String() string { + var opts []string + + if d.dc != "" { + opts = append(opts, fmt.Sprintf("dc=%s", d.dc)) + } + + sort.Strings(opts) + return fmt.Sprintf("node(%s)", strings.Join(opts, "&")) +} + +func (d *nodesQuery) Stop() { + close(d.stopCh) +} + +// ByID is a sortable slice of Node +type ByID []*dep.Node + +// Len, Swap, and Less are used to implement the sort.Sort interface. +func (s ByID) Len() int { return len(s) } +func (s ByID) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s ByID) Less(i, j int) bool { + return s[i].ID < s[j].ID +} diff --git a/templates/tftmpl/tmplfunc/nodes_test.go b/templates/tftmpl/tmplfunc/nodes_test.go new file mode 100644 index 000000000..977e14b6b --- /dev/null +++ b/templates/tftmpl/tmplfunc/nodes_test.go @@ -0,0 +1,141 @@ +package tmplfunc + +import ( + "testing" + + "github.com/hashicorp/consul-terraform-sync/testutils" + consulapi "github.com/hashicorp/consul/api" + "github.com/hashicorp/hcat/dep" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNewNodesQuery(t *testing.T) { + cases := []struct { + name string + opts []string + exp *nodesQuery + err bool + }{ + { + "no opts", + []string{}, + &nodesQuery{}, + false, + }, + { + "datacenter", + []string{"datacenter=dc2"}, + &nodesQuery{ + dc: "dc2", + }, + false, + }, + { + "invalid option", + []string{"hello=world"}, + nil, + true, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + act, err := newNodesQuery(tc.opts) + if tc.err { + assert.Error(t, err) + return + } + + if act != nil { + act.stopCh = nil + } + + assert.NoError(t, err, err) + assert.Equal(t, tc.exp, act) + }) + } +} + +func TestNodesQuery_String(t *testing.T) { + cases := []struct { + name string + i []string + exp string + }{ + { + + "datacenter", + []string{"datacenter=dc2"}, + "node(dc=dc2)", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + d, err := newNodesQuery(tc.i) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, tc.exp, d.String()) + }) + } +} + +func TestNodesQuery_Fetch(t *testing.T) { + srv := testutils.NewTestConsulServer(t, testutils.TestConsulServerConfig{}) + defer srv.Stop() + + consulConfig := consulapi.DefaultConfig() + consulConfig.Address = srv.HTTPAddr + client, err := consulapi.NewClient(consulConfig) + require.NoError(t, err, "failed to make consul client") + + srv.WaitForLeader(t) + + cases := []struct { + name string + i []string + expected []*dep.Node + }{ + { + "empty", + []string{}, + []*dep.Node{ + { + Address: "127.0.0.1", + Datacenter: "dc1", + TaggedAddresses: map[string]string{ + "lan": "127.0.0.1", + "lan_ipv4": "127.0.0.1", + "wan": "127.0.0.1", + "wan_ipv4": "127.0.0.1", + }, + Meta: map[string]string{ + "consul-network-segment": "", + }, + }, + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + d, err := newNodesQuery(tc.i) + require.NoError(t, err) + + a, _, err := d.Fetch(&testClient{consul: client}) + require.NoError(t, err) + actual, ok := a.([]*dep.Node) + require.True(t, ok) + require.Equal(t, len(tc.expected), len(actual)) + + for i, actualNode := range actual { + assert.Equal(t, tc.expected[i].Address, actualNode.Address) + assert.Equal(t, tc.expected[i].Datacenter, actualNode.Datacenter) + assert.Equal(t, tc.expected[i].TaggedAddresses, actualNode.TaggedAddresses) + assert.Equal(t, tc.expected[i].Meta, actualNode.Meta) + } + }) + } +} diff --git a/templates/tftmpl/tmplfunc/tmpl_func.go b/templates/tftmpl/tmplfunc/tmpl_func.go index d0db7d9e8..62568caae 100644 --- a/templates/tftmpl/tmplfunc/tmpl_func.go +++ b/templates/tftmpl/tmplfunc/tmpl_func.go @@ -21,11 +21,13 @@ func HCLMap(meta ServicesMeta) template.FuncMap { tmplFuncs := hcat.FuncMapConsulV1() tmplFuncs["catalogServicesRegistration"] = catalogServicesRegistrationFunc tmplFuncs["servicesRegex"] = servicesRegexFunc + tmplFuncs["nodes"] = nodesFunc tmplFuncs["indent"] = tfunc.Helpers()["indent"] tmplFuncs["subtract"] = tfunc.Math()["subtract"] tmplFuncs["joinStrings"] = joinStringsFunc tmplFuncs["HCLService"] = hclServiceFunc(meta) tmplFuncs["HCLServiceTags"] = hclServiceTagsFunc() + tmplFuncs["HCLNode"] = hclNodeFunc return tmplFuncs }