diff --git a/Makefile b/Makefile index 7b8b0749f..7bc333993 100644 --- a/Makefile +++ b/Makefile @@ -127,7 +127,8 @@ ifdef TEST @echo Running test $(TEST) KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test $(GO_FLAGS) ./apis/... ./pkg/... ./test/yq/... ./controllers/... -run="$(TEST)" -coverprofile cover.out else - KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test $(GO_FLAGS) ./apis/... ./pkg/... ./test/yq/... ./controllers/... -coverprofile cover.out + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test -v -run ^TestConfigMapController$ $(GO_FLAGS) ./controllers/replication/... -coverprofile cover.out + # KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test $(GO_FLAGS) ./apis/... ./pkg/... ./test/yq/... ./controllers/... -coverprofile cover.out endif PHONY: e2e-test diff --git a/PROJECT b/PROJECT index d0db36cb9..5ff8c7c74 100644 --- a/PROJECT +++ b/PROJECT @@ -46,6 +46,14 @@ resources: kind: ReplicatedSecret path: github.com/k8ssandra/k8ssandra-operator/apis/replication/v1alpha1 version: v1alpha1 +- api: + crdVersion: v1 + namespaced: true + domain: k8ssandra.io + group: replication + kind: ReplicatedConfigMap + path: github.com/k8ssandra/k8ssandra-operator/apis/replication/v1alpha1 + version: v1alpha1 - api: crdVersion: v1 namespaced: true diff --git a/apis/config/v1beta1/zz_generated.deepcopy.go b/apis/config/v1beta1/zz_generated.deepcopy.go index f0bc5e668..08c46aecf 100644 --- a/apis/config/v1beta1/zz_generated.deepcopy.go +++ b/apis/config/v1beta1/zz_generated.deepcopy.go @@ -2,7 +2,7 @@ // +build !ignore_autogenerated /* -Copyright 2021. +Copyright 2022. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/apis/k8ssandra/v1alpha1/zz_generated.deepcopy.go b/apis/k8ssandra/v1alpha1/zz_generated.deepcopy.go index b47345ece..c09fdd9e6 100644 --- a/apis/k8ssandra/v1alpha1/zz_generated.deepcopy.go +++ b/apis/k8ssandra/v1alpha1/zz_generated.deepcopy.go @@ -2,7 +2,7 @@ // +build !ignore_autogenerated /* -Copyright 2021. +Copyright 2022. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/apis/medusa/v1alpha1/zz_generated.deepcopy.go b/apis/medusa/v1alpha1/zz_generated.deepcopy.go index 77d5c390d..66f2943dd 100644 --- a/apis/medusa/v1alpha1/zz_generated.deepcopy.go +++ b/apis/medusa/v1alpha1/zz_generated.deepcopy.go @@ -2,7 +2,7 @@ // +build !ignore_autogenerated /* -Copyright 2021. +Copyright 2022. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/apis/reaper/v1alpha1/zz_generated.deepcopy.go b/apis/reaper/v1alpha1/zz_generated.deepcopy.go index 1d914d0b1..18ea351d3 100644 --- a/apis/reaper/v1alpha1/zz_generated.deepcopy.go +++ b/apis/reaper/v1alpha1/zz_generated.deepcopy.go @@ -2,7 +2,7 @@ // +build !ignore_autogenerated /* -Copyright 2021. +Copyright 2022. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/apis/replication/v1alpha1/replicatedconfigmap_types.go b/apis/replication/v1alpha1/replicatedconfigmap_types.go new file mode 100644 index 000000000..40299555d --- /dev/null +++ b/apis/replication/v1alpha1/replicatedconfigmap_types.go @@ -0,0 +1,63 @@ +/* +Copyright 2021. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +// ReplicatedConfigMapSpec defines the desired state of ReplicatedConfigMap +type ReplicatedConfigMapSpec struct { + *ReplicatedResourceSpec `json:",inline"` +} + +// ReplicatedConfigMapStatus defines the observed state of ReplicatedConfigMap +type ReplicatedConfigMapStatus struct { + // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + // Important: Run "make" to regenerate code after modifying this file + + // +optional + Conditions []ReplicationCondition `json:"conditions,omitempty"` +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status + +// ReplicatedConfigMap is the Schema for the ReplicatedConfigMaps API +type ReplicatedConfigMap struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec ReplicatedConfigMapSpec `json:"spec,omitempty"` + Status ReplicatedConfigMapStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// ReplicatedConfigMapList contains a list of ReplicatedConfigMap +type ReplicatedConfigMapList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []ReplicatedConfigMap `json:"items"` +} + +func init() { + SchemeBuilder.Register(&ReplicatedConfigMap{}, &ReplicatedConfigMapList{}) +} diff --git a/apis/replication/v1alpha1/replicatedresource_types.go b/apis/replication/v1alpha1/replicatedresource_types.go new file mode 100644 index 000000000..6b9afa07e --- /dev/null +++ b/apis/replication/v1alpha1/replicatedresource_types.go @@ -0,0 +1,59 @@ +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// type ReplicatedResource interface { +// ReplicationTargets() []ReplicationTarget +// Selector() *metav1.LabelSelector +// } + +type ReplicatedResourceSpec struct { + // TODO Add Kind + ApiVersion? + + // Selector defines which secrets are replicated. If left empty, all the secrets are replicated + // +optional + Selector *metav1.LabelSelector `json:"selector,omitempty"` + // TargetContexts indicates the target clusters to which the secrets are replicated to. If empty, no clusters are targeted + // +optional + ReplicationTargets []ReplicationTarget `json:"replicationTargets,omitempty"` +} + +type ReplicationTarget struct { + // TODO Implement at some point + // Namespace to replicate the data to in the target cluster. If left empty, current namespace is used. + // +optional + Namespace string `json:"namespace,omitempty"` + + // K8sContextName defines the target cluster name as set in the ClientConfig. If left empty, current cluster is assumed + // +optional + K8sContextName string `json:"k8sContextName,omitempty"` + + // TODO Add label selector for clusters (from ClientConfigs) + // Selector defines which clusters are targeted. + // +optional + // Selector *metav1.LabelSelector `json:"selector,omitempty"` +} + +type ReplicationConditionType string + +const ( + ReplicationDone ReplicationConditionType = "Replication" +) + +type ReplicationCondition struct { + // Cluster + Cluster string `json:"cluster"` + + // Type of condition + Type ReplicationConditionType `json:"type"` + + // Status of the replication to target cluster + Status corev1.ConditionStatus `json:"status"` + + // LastTransitionTime is the last time the condition transited from one status to another. + // +optional + LastTransitionTime *metav1.Time `json:"lastTransitionTime,omitempty"` +} diff --git a/apis/replication/v1alpha1/replicatedsecret_types.go b/apis/replication/v1alpha1/replicatedsecret_types.go index aaee871e8..1149fd9d5 100644 --- a/apis/replication/v1alpha1/replicatedsecret_types.go +++ b/apis/replication/v1alpha1/replicatedsecret_types.go @@ -17,72 +17,40 @@ limitations under the License. package v1alpha1 import ( - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! -// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. +// var _ ReplicatedResource = &ReplicatedSecretSpec{} + +// type ReplicatedResource interface { +// *ReplicatedResourceSpec +// } // ReplicatedSecretSpec defines the desired state of ReplicatedSecret type ReplicatedSecretSpec struct { - // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster - // Important: Run "make" to regenerate code after modifying this file - - // Selector defines which secrets are replicated. If left empty, all the secrets are replicated - // +optional - Selector *metav1.LabelSelector `json:"selector,omitempty"` - // TargetContexts indicates the target clusters to which the secrets are replicated to. If empty, no clusters are targeted - // +optional - ReplicationTargets []ReplicationTarget `json:"replicationTargets,omitempty"` + *ReplicatedResourceSpec `json:",inline"` + // // Selector defines which secrets are replicated. If left empty, all the secrets are replicated + // // +optional + // Selector *metav1.LabelSelector `json:"selector,omitempty"` + // // TargetContexts indicates the target clusters to which the secrets are replicated to. If empty, no clusters are targeted + // // +optional + // ReplicationTargets []ReplicationTarget `json:"replicationTargets,omitempty"` } -type ReplicationTarget struct { - // TODO Implement at some point - // Namespace to replicate the data to in the target cluster. If left empty, current namespace is used. - // +optional - Namespace string `json:"namespace,omitempty"` +// func (r *ReplicatedSecretSpec) ReplicationTargets() []ReplicationTarget { +// return r.ReplicationTargets +// } - // K8sContextName defines the target cluster name as set in the ClientConfig. If left empty, current cluster is assumed - // +optional - K8sContextName string `json:"k8sContextName,omitempty"` - - // TODO Add label selector for clusters (from ClientConfigs) - // Selector defines which clusters are targeted. - // +optional - // Selector *metav1.LabelSelector `json:"selector,omitempty"` -} +// func (r *ReplicatedSecretSpec) Selector() *metav1.LabelSelector { +// return r.Selector +// } // ReplicatedSecretStatus defines the observed state of ReplicatedSecret type ReplicatedSecretStatus struct { - // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster - // Important: Run "make" to regenerate code after modifying this file - // +optional Conditions []ReplicationCondition `json:"conditions,omitempty"` } -type ReplicationConditionType string - -const ( - ReplicationDone ReplicationConditionType = "Replication" -) - -type ReplicationCondition struct { - // Cluster - Cluster string `json:"cluster"` - - // Type of condition - Type ReplicationConditionType `json:"type"` - - // Status of the replication to target cluster - Status corev1.ConditionStatus `json:"status"` - - // LastTransitionTime is the last time the condition transited from one status to another. - // +optional - LastTransitionTime *metav1.Time `json:"lastTransitionTime,omitempty"` -} - //+kubebuilder:object:root=true //+kubebuilder:subresource:status diff --git a/apis/replication/v1alpha1/zz_generated.deepcopy.go b/apis/replication/v1alpha1/zz_generated.deepcopy.go index 710fceada..b6d994de8 100644 --- a/apis/replication/v1alpha1/zz_generated.deepcopy.go +++ b/apis/replication/v1alpha1/zz_generated.deepcopy.go @@ -2,7 +2,7 @@ // +build !ignore_autogenerated /* -Copyright 2021. +Copyright 2022. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -26,6 +26,132 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ReplicatedConfigMap) DeepCopyInto(out *ReplicatedConfigMap) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReplicatedConfigMap. +func (in *ReplicatedConfigMap) DeepCopy() *ReplicatedConfigMap { + if in == nil { + return nil + } + out := new(ReplicatedConfigMap) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ReplicatedConfigMap) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ReplicatedConfigMapList) DeepCopyInto(out *ReplicatedConfigMapList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ReplicatedConfigMap, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReplicatedConfigMapList. +func (in *ReplicatedConfigMapList) DeepCopy() *ReplicatedConfigMapList { + if in == nil { + return nil + } + out := new(ReplicatedConfigMapList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ReplicatedConfigMapList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ReplicatedConfigMapSpec) DeepCopyInto(out *ReplicatedConfigMapSpec) { + *out = *in + if in.ReplicatedResourceSpec != nil { + in, out := &in.ReplicatedResourceSpec, &out.ReplicatedResourceSpec + *out = new(ReplicatedResourceSpec) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReplicatedConfigMapSpec. +func (in *ReplicatedConfigMapSpec) DeepCopy() *ReplicatedConfigMapSpec { + if in == nil { + return nil + } + out := new(ReplicatedConfigMapSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ReplicatedConfigMapStatus) DeepCopyInto(out *ReplicatedConfigMapStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]ReplicationCondition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReplicatedConfigMapStatus. +func (in *ReplicatedConfigMapStatus) DeepCopy() *ReplicatedConfigMapStatus { + if in == nil { + return nil + } + out := new(ReplicatedConfigMapStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ReplicatedResourceSpec) DeepCopyInto(out *ReplicatedResourceSpec) { + *out = *in + if in.Selector != nil { + in, out := &in.Selector, &out.Selector + *out = new(v1.LabelSelector) + (*in).DeepCopyInto(*out) + } + if in.ReplicationTargets != nil { + in, out := &in.ReplicationTargets, &out.ReplicationTargets + *out = make([]ReplicationTarget, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReplicatedResourceSpec. +func (in *ReplicatedResourceSpec) DeepCopy() *ReplicatedResourceSpec { + if in == nil { + return nil + } + out := new(ReplicatedResourceSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ReplicatedSecret) DeepCopyInto(out *ReplicatedSecret) { *out = *in @@ -88,16 +214,11 @@ func (in *ReplicatedSecretList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ReplicatedSecretSpec) DeepCopyInto(out *ReplicatedSecretSpec) { *out = *in - if in.Selector != nil { - in, out := &in.Selector, &out.Selector - *out = new(v1.LabelSelector) + if in.ReplicatedResourceSpec != nil { + in, out := &in.ReplicatedResourceSpec, &out.ReplicatedResourceSpec + *out = new(ReplicatedResourceSpec) (*in).DeepCopyInto(*out) } - if in.ReplicationTargets != nil { - in, out := &in.ReplicationTargets, &out.ReplicationTargets - *out = make([]ReplicationTarget, len(*in)) - copy(*out, *in) - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReplicatedSecretSpec. diff --git a/apis/stargate/v1alpha1/zz_generated.deepcopy.go b/apis/stargate/v1alpha1/zz_generated.deepcopy.go index aaf636d23..bcc319019 100644 --- a/apis/stargate/v1alpha1/zz_generated.deepcopy.go +++ b/apis/stargate/v1alpha1/zz_generated.deepcopy.go @@ -2,7 +2,7 @@ // +build !ignore_autogenerated /* -Copyright 2021. +Copyright 2022. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/apis/telemetry/v1alpha1/zz_generated.deepcopy.go b/apis/telemetry/v1alpha1/zz_generated.deepcopy.go index 6eb70e8d6..ace85872a 100644 --- a/apis/telemetry/v1alpha1/zz_generated.deepcopy.go +++ b/apis/telemetry/v1alpha1/zz_generated.deepcopy.go @@ -2,7 +2,7 @@ // +build !ignore_autogenerated /* -Copyright 2021. +Copyright 2022. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/config/crd/bases/replication.k8ssandra.io_replicatedconfigmaps.yaml b/config/crd/bases/replication.k8ssandra.io_replicatedconfigmaps.yaml new file mode 100644 index 000000000..1ac159b2c --- /dev/null +++ b/config/crd/bases/replication.k8ssandra.io_replicatedconfigmaps.yaml @@ -0,0 +1,139 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.8.0 + creationTimestamp: null + name: replicatedconfigmaps.replication.k8ssandra.io +spec: + group: replication.k8ssandra.io + names: + kind: ReplicatedConfigMap + listKind: ReplicatedConfigMapList + plural: replicatedconfigmaps + singular: replicatedconfigmap + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: ReplicatedConfigMap is the Schema for the ReplicatedConfigMaps + API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ReplicatedConfigMapSpec defines the desired state of ReplicatedConfigMap + properties: + replicationTargets: + description: TargetContexts indicates the target clusters to which + the secrets are replicated to. If empty, no clusters are targeted + items: + properties: + k8sContextName: + description: K8sContextName defines the target cluster name + as set in the ClientConfig. If left empty, current cluster + is assumed + type: string + namespace: + description: TODO Implement at some point Namespace to replicate + the data to in the target cluster. If left empty, current + namespace is used. + type: string + type: object + type: array + selector: + description: Selector defines which secrets are replicated. If left + empty, all the secrets are replicated + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + type: object + status: + description: ReplicatedConfigMapStatus defines the observed state of ReplicatedConfigMap + properties: + conditions: + items: + properties: + cluster: + description: Cluster + type: string + lastTransitionTime: + description: LastTransitionTime is the last time the condition + transited from one status to another. + format: date-time + type: string + status: + description: Status of the replication to target cluster + type: string + type: + description: Type of condition + type: string + required: + - cluster + - status + - type + type: object + type: array + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 0bbeae636..ed17949fe 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -6,6 +6,7 @@ resources: - bases/stargate.k8ssandra.io_stargates.yaml - bases/config.k8ssandra.io_clientconfigs.yaml - bases/replication.k8ssandra.io_replicatedsecrets.yaml +- bases/replication.k8ssandra.io_replicatedconfigmaps.yaml - bases/reaper.k8ssandra.io_reapers.yaml - bases/medusa.k8ssandra.io_cassandrabackups.yaml - bases/medusa.k8ssandra.io_cassandrarestores.yaml diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 820f800a9..b13f88e84 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -70,6 +70,17 @@ rules: - patch - update - watch +- apiGroups: + - "" + resources: + - configMaps + verbs: + - create + - delete + - get + - list + - update + - watch - apiGroups: - "" resources: @@ -233,6 +244,31 @@ rules: - get - patch - update +- apiGroups: + - replication.k8ssandra.io + resources: + - replicatedConfigMaps + verbs: + - create + - delete + - get + - list + - update + - watch +- apiGroups: + - replication.k8ssandra.io + resources: + - replicatedConfigMaps/finalizers + verbs: + - update +- apiGroups: + - replication.k8ssandra.io + resources: + - replicatedConfigMaps/status + verbs: + - get + - patch + - update - apiGroups: - replication.k8ssandra.io resources: diff --git a/controllers/k8ssandra/k8ssandracluster_controller_test.go b/controllers/k8ssandra/k8ssandracluster_controller_test.go index d194b48d2..9b29835ff 100644 --- a/controllers/k8ssandra/k8ssandracluster_controller_test.go +++ b/controllers/k8ssandra/k8ssandracluster_controller_test.go @@ -930,10 +930,12 @@ func createReplicatedSecret(ctx context.Context, t *testing.T, f *framework.Fram Name: kcKey.Name, }, Spec: replicationapi.ReplicatedSecretSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: labels.ManagedByLabels(kcKey), + ReplicatedResourceSpec: &replicationapi.ReplicatedResourceSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: labels.ManagedByLabels(kcKey), + }, + ReplicationTargets: []replicationapi.ReplicationTarget{}, }, - ReplicationTargets: []replicationapi.ReplicationTarget{}, }, Status: replicationapi.ReplicatedSecretStatus{ Conditions: []replicationapi.ReplicationCondition{ diff --git a/controllers/replication/common.go b/controllers/replication/common.go new file mode 100644 index 000000000..d1d3a1fe0 --- /dev/null +++ b/controllers/replication/common.go @@ -0,0 +1,84 @@ +package replication + +import ( + "context" + "strings" + + coreapi "github.com/k8ssandra/k8ssandra-operator/apis/k8ssandra/v1alpha1" + "github.com/k8ssandra/k8ssandra-operator/pkg/utils" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// Move to pkg when done +func objectRequiresUpdate(source, dest client.Object) bool { + // In case we target the same cluster + if source.GetUID() == dest.GetUID() { + return false + } + + if srcHash, found := source.GetAnnotations()[coreapi.ResourceHashAnnotation]; found { + // Get dest hash value + destHash, destFound := dest.GetAnnotations()[coreapi.ResourceHashAnnotation] + if !destFound { + return true + } + + hash := getObjectHash(dest) + if destHash != hash { + // Destination data did not match destination hash + return true + } + + return srcHash != destHash + } + return false +} + +func syncMetadata(src, dest *metav1.ObjectMeta) { + // sync annotations, src is more important + for k, v := range src.GetAnnotations() { + if !datacenterSpecific(k) { + metav1.SetMetaDataAnnotation(dest, k, v) + dest.Annotations[k] = v + } + } + + // sync labels, src is more important + for k, v := range src.Labels { + if !datacenterSpecific(k) { + metav1.SetMetaDataLabel(dest, k, v) + } + } +} + +// filterValue verifies the annotation is not something datacenter specific +func datacenterSpecific(key string) bool { + return strings.HasPrefix(key, "cassandra.datastax.com/") +} + +func verifyHashAnnotation(ctx context.Context, localClient client.Client, sec client.Object) error { + hash := getObjectHash(sec) + if sec.GetAnnotations() == nil { + sec.SetAnnotations(make(map[string]string)) + } + if existingHash, found := sec.GetAnnotations()[coreapi.ResourceHashAnnotation]; !found || (existingHash != hash) { + sec.GetAnnotations()[coreapi.ResourceHashAnnotation] = hash + return localClient.Update(ctx, sec) + } + return nil +} + +func getObjectHash(target client.Object) string { + var obj interface{} + switch v := target.(type) { + case *corev1.ConfigMap: + obj = v.Data + case *corev1.Secret: + obj = v.Data + } + hash := utils.DeepHashString(obj) + return hash +} diff --git a/controllers/replication/resource_controller.go b/controllers/replication/resource_controller.go new file mode 100644 index 000000000..467f21038 --- /dev/null +++ b/controllers/replication/resource_controller.go @@ -0,0 +1,407 @@ +package replication + +import ( + "context" + "fmt" + "sync" + + api "github.com/k8ssandra/k8ssandra-operator/apis/replication/v1alpha1" + "github.com/k8ssandra/k8ssandra-operator/pkg/clientcache" + "github.com/k8ssandra/k8ssandra-operator/pkg/config" + "github.com/k8ssandra/k8ssandra-operator/pkg/secret" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + apimeta "k8s.io/apimachinery/pkg/api/meta" +) + +// We need rights to update the target cluster's ConfigMaps, not necessarily this cluster +// +kubebuilder:rbac:groups=core,namespace="k8ssandra",resources=configMaps,verbs=get;list;watch;update;create;delete +// +kubebuilder:rbac:groups=replication.k8ssandra.io,namespace="k8ssandra",resources=replicatedConfigMaps,verbs=get;list;watch;update;create;delete +// +kubebuilder:rbac:groups=replication.k8ssandra.io,namespace="k8ssandra",resources=replicatedConfigMaps/finalizers,verbs=update +// +kubebuilder:rbac:groups=replication.k8ssandra.io,namespace="k8ssandra",resources=replicatedConfigMaps/status,verbs=get;update;patch + +type ConfigMapSyncController struct { + *config.ReconcilerConfig + ClientCache *clientcache.ClientCache + // TODO We need a better structure for empty selectors (match whole kind) + WatchNamespaces []string + selectorMutex sync.RWMutex + selectors map[types.NamespacedName]labels.Selector +} + +type SelectorCache struct { + // Mutex + // Kind + types.NamespacedName => labels.Selector (ObjectMeta?) +} + +func (s *ConfigMapSyncController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + logger := log.FromContext(ctx) + + localClient := s.ClientCache.GetLocalClient() + + logger.Info("ConfigMapSyncController::Starting reconciliation", "key", req.NamespacedName) + + // TODO Should not be ReplicatedConfigMap, but ReplicatedResource with enough information to get the type + rsec := &api.ReplicatedConfigMap{} + if err := localClient.Get(ctx, req.NamespacedName, rsec); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + // Deletion and finalizer logic + if rsec.GetDeletionTimestamp() != nil { + return s.deletionProcess(ctx, rsec) + } + + if !controllerutil.ContainsFinalizer(rsec, replicatedResourceFinalizer) { + controllerutil.AddFinalizer(rsec, replicatedResourceFinalizer) + if err := localClient.Update(ctx, rsec); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{Requeue: true}, nil + } + + // Add the new matcher rules also to our cache if not found + selector, err := metav1.LabelSelectorAsSelector(rsec.Spec.Selector) + if err != nil { + return reconcile.Result{}, err + } + + // Update the selector in cache always (comparing is pointless) + s.selectorMutex.Lock() + s.selectors[req.NamespacedName] = selector + s.selectorMutex.Unlock() + + // TODO This could be a func in the struct to do fetching + + // Fetch all the ConfigMaps that match the ReplicatedConfigMap's rules + // configMaps, err := s.fetchAllMatchingConfigMaps(ctx, selector) + // if err != nil { + // return reconcile.Result{}, err + // } + + // TODO ConfigMapList should come from a function that matches the known type (or the API type) + items, err := s.fetchAllMatchingObjects(ctx, selector, &corev1.ConfigMapList{}) + if err != nil { + return reconcile.Result{}, err + } + + // Verify objects have up-to-date hashes + for i := range items { + sec := items[i] + if err := verifyHashAnnotation(ctx, s.ClientCache.GetLocalClient(), sec.(client.Object)); err != nil { + return reconcile.Result{}, err + } + } + + // For status updates + patch := client.MergeFrom(rsec.DeepCopy()) + rsec.Status.Conditions = make([]api.ReplicationCondition, 0, len(rsec.Spec.ReplicationTargets)) + + for _, target := range rsec.Spec.ReplicationTargets { + // Even if ReplicationTarget includes local client - remove it (it will cause errors) + // Only replicate to clusters that are in the ReplicatedConfigMap's context + var remoteClient client.Client + if target.K8sContextName == "" { + remoteClient = localClient + } else { + remoteClient, err = s.ClientCache.GetRemoteClient(target.K8sContextName) + if err != nil { + logger.Error(err, "Failed to fetch remote client for managed cluster", "ReplicatedConfigMap", req.NamespacedName, "TargetContext", target) + return ctrl.Result{Requeue: true}, err + } + } + + cond := api.ReplicationCondition{ + Cluster: target.K8sContextName, + Type: api.ReplicationDone, + } + + TargetConfigMaps: + // Iterate all the matching ConfigMaps + for i := range items { + sec := items[i].(client.Object) + namespace := "" + if target.Namespace == "" { + namespace = sec.GetNamespace() + } else { + namespace = target.Namespace + } + // TODO Need to probably cast here unless we can use some other type.. + fetchedConfigMap := &corev1.ConfigMap{} + if err = remoteClient.Get(ctx, types.NamespacedName{Name: sec.GetName(), Namespace: namespace}, fetchedConfigMap); err != nil { + if errors.IsNotFound(err) { + logger.Info("Copying ConfigMap to target cluster", "ConfigMap", sec.GetName(), "TargetContext", target) + // Create it + copiedConfigMap := sec.DeepCopyObject().(client.Object) + copiedConfigMap.SetNamespace(namespace) + copiedConfigMap.SetResourceVersion("") + copiedConfigMap.SetOwnerReferences([]metav1.OwnerReference{}) + if err = remoteClient.Create(ctx, copiedConfigMap); err != nil { + logger.Error(err, "Failed to sync ConfigMap to target cluster", "ConfigMap", copiedConfigMap.GetName(), "TargetContext", target) + break TargetConfigMaps + } + continue + } + logger.Error(err, "Failed to fetch ConfigMap from target cluster", "ConfigMap", fetchedConfigMap.Name, "TargetContext", target) + break TargetConfigMaps + } + + if fetchedConfigMap.Immutable != nil && *fetchedConfigMap.Immutable { + err := fmt.Errorf("target ConfigMap is immutable") + logger.Error(err, "Failed to modify target ConfigMap, ConfigMap is set to immutable", "ConfigMap", fetchedConfigMap.Name, "TargetContext", target) + break TargetConfigMaps + } + + if objectRequiresUpdate(sec, fetchedConfigMap) { + logger.Info("Modifying ConfigMap in target cluster", "ConfigMap", sec.GetName(), "TargetContext", target) + // TODO Need sync to have a type here .. + origConfigMap := sec.(*corev1.ConfigMap) + syncConfigMaps(origConfigMap, fetchedConfigMap) + if err = remoteClient.Update(ctx, fetchedConfigMap); err != nil { + logger.Error(err, "Failed to sync target ConfigMap for matching payloads", "ConfigMap", fetchedConfigMap.Name, "TargetContext", target) + break TargetConfigMaps + } + } + } + if err != nil { + cond.Status = corev1.ConditionFalse + } else { + cond.Status = corev1.ConditionTrue + } + + timeNow := metav1.Now() + cond.LastTransitionTime = &timeNow + rsec.Status.Conditions = append(rsec.Status.Conditions, cond) + } + + // Update the ReplicatedConfigMap's Status + err = localClient.Status().Patch(ctx, rsec, patch) + if err != nil { + logger.Error(err, "Failed to update replicated ConfigMap last transition time", "ReplicatedConfigMap", req.NamespacedName) + return ctrl.Result{}, err + } + + // If any cluster had failed state, retry + for _, cond := range rsec.Status.Conditions { + if cond.Status == corev1.ConditionFalse { + return ctrl.Result{}, fmt.Errorf("replication failed") + } + } + return ctrl.Result{}, err +} + +// TODO Refactor to make it common with SecretController +// Fetcher is the problematic one, are there any others? +func (s *ConfigMapSyncController) deletionProcess(ctx context.Context, rsec *api.ReplicatedConfigMap) (ctrl.Result, error) { + localClient := s.ClientCache.GetLocalClient() + logger := log.FromContext(ctx) + + namespacedName := types.NamespacedName{Name: rsec.Name, Namespace: rsec.Namespace} + + if controllerutil.ContainsFinalizer(rsec, replicatedResourceFinalizer) { + logger.Info("Starting cleanup") + + // Fetch all ConfigMaps from managed cluster. + // Remove only those ConfigMaps which are not matched by any other ReplicatedConfigMap and do not have the orphan annotation + if val, found := rsec.GetAnnotations()[secret.OrphanResourceAnnotation]; !found || val != "true" { + logger.Info("Cleaning up all the replicated resources", "ReplicatedConfigMap", namespacedName) + selector, err := metav1.LabelSelectorAsSelector(rsec.Spec.Selector) + if err != nil { + logger.Error(err, "Failed to delete the replicated ConfigMap, defined labels are invalid", "ReplicatedConfigMap", namespacedName) + return reconcile.Result{}, err + } + + // TODO Needs a common type here for the ConfigMapList (or a separate fetcher?) + configMaps, err := s.fetchAllMatchingObjects(ctx, selector, &corev1.ConfigMapList{}) + // configMaps, err := s.fetchAllMatchingConfigMaps(ctx, selector) + if err != nil { + logger.Error(err, "Failed to fetch the replicated ConfigMaps to cleanup", "ReplicatedConfigMap", namespacedName) + return reconcile.Result{}, err + } + + configMapsToDelete := make([]client.Object, 0, len(configMaps)) + + s.selectorMutex.RLock() + + ConfigMapsToCheck: + for _, sec := range configMaps { + accessor, err := apimeta.Accessor(sec) + if err != nil { + return reconcile.Result{}, err + } + key := types.NamespacedName{Name: accessor.GetName(), Namespace: accessor.GetNamespace()} + logger.Info("Checking ConfigMap", "key", key) + for k, v := range s.selectors { + if k.Namespace != key.Namespace { + logger.Info("Skipping ConfigMap", "key", key, "namespace", k.Namespace) + continue + } + if k == namespacedName { + // This is the ReplicatedConfigMap that will be deleted, we don't want its rules to match + continue + } + + if val, found := accessor.GetAnnotations()[secret.OrphanResourceAnnotation]; found && val == "true" { + // Managed cluster has orphan set to the ConfigMap, do not delete it from target clusters + continue ConfigMapsToCheck + } + + if v.Matches(labels.Set(accessor.GetLabels())) { + // Another Replication rule is matching this ConfigMap, do not delete it + logger.Info("Another replication rule matches ConfigMap", "key", key) + continue ConfigMapsToCheck + } + } + logger.Info("Preparing to delete ConfigMap", "key", key) + configMapsToDelete = append(configMapsToDelete, sec.DeepCopyObject().(client.Object)) + } + + s.selectorMutex.RUnlock() + + for _, target := range rsec.Spec.ReplicationTargets { + logger.Info("Deleting ConfigMaps for ReplicationTarget", "Target", target) + + // Only replicate to clusters that are in the ReplicatedConfigMap's context + remoteClient, err := s.ClientCache.GetRemoteClient(target.K8sContextName) + if err != nil { + logger.Error(err, "Failed to fetch remote client for managed cluster", "ReplicatedConfigMap", namespacedName, "TargetContext", target) + return ctrl.Result{}, err + } + for _, deleteKey := range configMapsToDelete { + logger.Info("Deleting ConfigMap", "key", client.ObjectKeyFromObject(deleteKey), + "Cluster", target.K8sContextName) + err = remoteClient.Delete(ctx, deleteKey) + if err != nil && !errors.IsNotFound(err) { + logger.Error(err, "Failed to remove ConfigMaps from target cluster", "ReplicatedConfigMap", namespacedName, "TargetContext", target) + return ctrl.Result{}, err + } + } + } + } + s.selectorMutex.Lock() + delete(s.selectors, namespacedName) + s.selectorMutex.Unlock() + controllerutil.RemoveFinalizer(rsec, replicatedResourceFinalizer) + err := localClient.Update(ctx, rsec) + if err != nil { + return ctrl.Result{Requeue: true}, err + } + } + return ctrl.Result{}, nil +} + +// TODO Change type to client.Object etc +func syncConfigMaps(src, dest *corev1.ConfigMap) { + origMeta := dest.ObjectMeta + src.DeepCopyInto(dest) + dest.ObjectMeta = origMeta + dest.OwnerReferences = []metav1.OwnerReference{} + + syncMetadata(&src.ObjectMeta, &dest.ObjectMeta) +} + +func (s *ConfigMapSyncController) fetchAllMatchingConfigMaps(ctx context.Context, selector labels.Selector) ([]corev1.ConfigMap, error) { + configMaps := &corev1.ConfigMapList{} + listOption := client.ListOptions{ + LabelSelector: selector, + } + err := s.ClientCache.GetLocalClient().List(ctx, configMaps, &listOption) + if err != nil { + return nil, err + } + + return configMaps.Items, nil +} + +func (s *ConfigMapSyncController) fetchAllMatchingObjects(ctx context.Context, selector labels.Selector, through client.ObjectList) ([]runtime.Object, error) { + listOption := client.ListOptions{ + LabelSelector: selector, + } + err := s.ClientCache.GetLocalClient().List(ctx, through, &listOption) + if err != nil { + return nil, err + } + + allItems, err := apimeta.ExtractList(through) + return allItems, err +} + +func (s *ConfigMapSyncController) SetupWithManager(mgr ctrl.Manager, clusters []cluster.Cluster) error { + err := s.initializeCache() + if err != nil { + return err + } + + cb := ctrl.NewControllerManagedBy(mgr). + For(&api.ReplicatedConfigMap{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})). + Watches(&source.Kind{Type: &corev1.ConfigMap{}}, handler.EnqueueRequestsFromMapFunc(s.replicaMatcher)) + + for _, c := range clusters { + cb = cb.Watches( + source.NewKindWithCache(&corev1.ConfigMap{}, c.GetCache()), + handler.EnqueueRequestsFromMapFunc(s.replicaMatcher)) + } + + return cb.Complete(s) +} + +// TODO Duplicate code with SecretController +func (s *ConfigMapSyncController) replicaMatcher(configMap client.Object) []reconcile.Request { + requests := []reconcile.Request{} + s.selectorMutex.RLock() + for k, v := range s.selectors { + if v.Matches(labels.Set(configMap.GetLabels())) { + requests = append(requests, reconcile.Request{NamespacedName: k}) + } + } + s.selectorMutex.RUnlock() + return requests +} + +// TODO Duplicate code with secretController +func (s *ConfigMapSyncController) initializeCache() error { + s.selectors = make(map[types.NamespacedName]labels.Selector) + localClient := s.ClientCache.GetLocalNonCacheClient() + + for _, namespace := range s.WatchNamespaces { + var err error + replicatedConfigMaps := api.ReplicatedConfigMapList{} + opts := make([]client.ListOption, 0, 1) + if namespace != "" { + opts = append(opts, client.InNamespace(namespace)) + } + err = localClient.List(context.Background(), &replicatedConfigMaps, opts...) + if err != nil { + return err + } + + for _, rsec := range replicatedConfigMaps.Items { + namespacedName := types.NamespacedName{Name: rsec.Name, Namespace: rsec.Namespace} + // Add the new matcher rules also to our cache if not found + selector, err := metav1.LabelSelectorAsSelector(rsec.Spec.Selector) + if err != nil { + return err + } + + s.selectorMutex.Lock() + s.selectors[namespacedName] = selector + s.selectorMutex.Unlock() + } + } + return nil +} diff --git a/controllers/replication/resource_controller_test.go b/controllers/replication/resource_controller_test.go new file mode 100644 index 000000000..cd9826c28 --- /dev/null +++ b/controllers/replication/resource_controller_test.go @@ -0,0 +1,271 @@ +package replication + +import ( + "context" + "fmt" + "testing" + "time" + + coreapi "github.com/k8ssandra/k8ssandra-operator/apis/k8ssandra/v1alpha1" + api "github.com/k8ssandra/k8ssandra-operator/apis/replication/v1alpha1" + "github.com/k8ssandra/k8ssandra-operator/pkg/clientcache" + "github.com/k8ssandra/k8ssandra-operator/pkg/config" + testutils "github.com/k8ssandra/k8ssandra-operator/pkg/test" + "github.com/k8ssandra/k8ssandra-operator/test/framework" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" + "sigs.k8s.io/controller-runtime/pkg/manager" +) + +func TestConfigMapController(t *testing.T) { + ctx := testutils.TestSetup(t) + ctx, cancel := context.WithCancel(ctx) + testEnv = &testutils.MultiClusterTestEnv{} + err := testEnv.Start(ctx, t, func(mgr manager.Manager, clientCache *clientcache.ClientCache, clusters []cluster.Cluster) error { + scheme = mgr.GetScheme() + logger = mgr.GetLogger() + return (&ConfigMapSyncController{ + ReconcilerConfig: config.InitConfig(), + ClientCache: clientCache, + }).SetupWithManager(mgr, clusters) + }) + if err != nil { + t.Fatalf("failed to start test environment: %s", err) + } + + defer testEnv.Stop(t) + defer cancel() + + // ConfigMap controller tests + t.Run("MultiClusterSyncConfigMapTest", testEnv.ControllerTest(ctx, copyConfigMapFromClusterToCluster)) +} + +// copyConfigMapsFromClusterToCluster Tests: +// * Copy configMap to another cluster (not existing one) +// * Modify the configMap in main cluster - see that it is updated to slave cluster also +// * Modify the configMap in the slave cluster - see that it is replaced by the main cluster data +// * Verify the status has been updated +func copyConfigMapFromClusterToCluster(t *testing.T, ctx context.Context, f *framework.Framework, namespace string) { + require := require.New(t) + // assert := assert.New(t) + var empty struct{} + + rsec := generateReplicatedConfigMap(f.K8sContext(1), namespace) + rsec.Name = "broke" + err := f.Client.Create(ctx, rsec) + require.NoError(err, "failed to create replicated configMap to main cluster") + + generatedConfigMaps := generateConfigMaps(namespace) + for i, s := range generatedConfigMaps { + s.Name = fmt.Sprintf("broken-configmap-%d", i) + err := f.Client.Create(ctx, s) + require.NoError(err, "failed to create configMap to main cluster") + } + + startTime := time.Now() + + t.Log("check that the configMaps were copied to other cluster(s)") + require.Eventually(func() bool { + return verifyConfigMapsMatch(t, ctx, f.Client, []string{f.K8sContext(targetCopyToCluster)}, map[string]struct{}{ + generatedConfigMaps[0].Name: empty, + }, generatedConfigMaps[0].Namespace) + }, timeout, interval) + + t.Log("check that configMap not match by replicated configMap was not copied") + require.Never(func() bool { + return verifyConfigMapsMatch(t, ctx, f.Client, []string{f.K8sContext(targetCopyToCluster)}, map[string]struct{}{ + generatedConfigMaps[1].Name: empty, + }, generatedConfigMaps[0].Namespace) + }, 3, interval) + + t.Log("check that nothing was copied to cluster not match by replicated configMap") + require.Never(func() bool { + return verifyConfigMapsMatch(t, ctx, f.Client, []string{f.K8sContext(targetNoCopyCluster)}, map[string]struct{}{ + generatedConfigMaps[0].Name: empty, + generatedConfigMaps[1].Name: empty, + }, generatedConfigMaps[0].Namespace) + }, 3, interval) + + t.Log("modify the configMap in the main cluster") + toModifyConfigMap := &corev1.ConfigMap{} + err = f.Client.Get(context.TODO(), types.NamespacedName{Name: generatedConfigMaps[0].Name, Namespace: namespace}, toModifyConfigMap) + require.NoError(err, "failed to fetch modified configMap from the main cluster") + toModifyConfigMap.Data["newKey"] = "my-new-value" + err = f.Client.Update(ctx, toModifyConfigMap) + require.NoError(err, "failed to modify configMap in the main cluster") + + t.Log("verify it was modified in the target cluster also") + require.Eventually(func() bool { + return verifyConfigMapsMatch(t, ctx, f.Client, []string{f.K8sContext(targetCopyToCluster)}, map[string]struct{}{ + generatedConfigMaps[0].Name: empty, + }, generatedConfigMaps[0].Namespace) + }, timeout, interval) + + t.Log("modify the configMap in target cluster") + modifierClient := testEnv.Clients[f.K8sContext(targetCopyToCluster)] + targetConfigMaps := &corev1.ConfigMapList{} + err = modifierClient.List(ctx, targetConfigMaps, client.InNamespace(generatedConfigMaps[0].Namespace)) + require.NoError(err) + for _, targetConfigMap := range targetConfigMaps.Items { + if targetConfigMap.Name == generatedConfigMaps[0].Name { + phantomConfigMap := targetConfigMap.DeepCopy() + phantomConfigMap.Data["be-gone-key"] = "my-phantom-value" + targetConfigMap.GetAnnotations()[coreapi.ResourceHashAnnotation] = "XXXXXX" + err = modifierClient.Update(ctx, phantomConfigMap) + require.NoError(err) + break + } + } + + t.Log("verify it was returned to original form") + require.Eventually(func() bool { + return verifyConfigMapsMatch(t, ctx, f.Client, []string{f.K8sContext(targetCopyToCluster)}, map[string]struct{}{ + generatedConfigMaps[0].Name: empty, + }, generatedConfigMaps[0].Namespace) + }, timeout, interval) + + t.Log("check status is set to complete") + // Get updated status + require.Eventually(func() bool { + updatedRSec := &api.ReplicatedConfigMap{} + err = f.Client.Get(context.TODO(), types.NamespacedName{Name: rsec.Name, Namespace: rsec.Namespace}, updatedRSec) + if err != nil { + return false + } + // require.NoError(err) + + // We only copy to a single target cluster + if len(updatedRSec.Status.Conditions) != 1 { + return false + } + for _, cond := range updatedRSec.Status.Conditions { + if !(cond.LastTransitionTime.After(startTime) && + cond.Cluster != "" && + cond.Type == api.ReplicationDone && + cond.Status == corev1.ConditionTrue) { + return false + } + } + return true + }, timeout, interval) + + t.Log("delete the replicated configMap") + err = f.Client.Delete(ctx, rsec) + require.NoError(err, "failed to delete replicated configMap from main cluster") + + t.Log("verify the replicated configMaps are gone from the remote cluster") + remoteClient := testEnv.Clients[f.K8sContext(targetCopyToCluster)] + require.Eventually(func() bool { + t.Logf("checking for configMap deletion: %v", types.NamespacedName{Name: generatedConfigMaps[0].Name, Namespace: rsec.Namespace}) + remoteConfigMap := &corev1.ConfigMap{} + err := remoteClient.Get(context.TODO(), types.NamespacedName{Name: generatedConfigMaps[0].Name, Namespace: rsec.Namespace}, remoteConfigMap) + if err != nil { + if !errors.IsNotFound(err) { + t.Logf("Failed to get configMap: %v", err) + } + return errors.IsNotFound(err) + } + return false + }, timeout, interval) +} + +// verifySecretsMatch checks that the same secret is copied to other clusters +func verifyConfigMapsMatch(t *testing.T, ctx context.Context, localClient client.Client, remoteClusters []string, secrets map[string]struct{}, namespace string) bool { + configMapList := &corev1.ConfigMapList{} + err := localClient.List(ctx, configMapList, client.InNamespace(namespace)) + if err != nil { + return false + } + + for _, remoteCluster := range remoteClusters { + testClient := testEnv.Clients[remoteCluster] + + configMapList := &corev1.ConfigMapList{} + err := testClient.List(ctx, configMapList, client.InNamespace(namespace)) + if err != nil { + return false + } + + for _, s := range configMapList.Items { + if _, exists := secrets[s.Name]; exists { + // Find the corresponding item from targetSecretList - or fail if it's not there + found := false + for _, ts := range configMapList.Items { + if s.Name == ts.Name { + found = true + if s.GetAnnotations()[coreapi.ResourceHashAnnotation] != ts.GetAnnotations()[coreapi.ResourceHashAnnotation] { + return false + } + break + } + } + if !found { + return false + } + } + } + } + + return true +} + +func generateConfigMaps(namespace string) []*corev1.ConfigMap { + return []*corev1.ConfigMap{ + { + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: "test-configMap-first", + Labels: map[string]string{ + "configMap-controller": "test", + coreapi.K8ssandraClusterNamespaceLabel: namespace, + coreapi.K8ssandraClusterNameLabel: "k8sssandra", + }, + }, + Data: map[string]string{ + "key": "value", + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: "test-configMap-second", + Labels: map[string]string{ + "configMap-controller": "nomatch", + }, + }, + Data: map[string]string{ + "key": "value", + }, + }, + } +} + +func generateReplicatedConfigMap(k8sContext, namespace string) *api.ReplicatedConfigMap { + return &api.ReplicatedConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: "fetch-configMaps", + }, + Spec: api.ReplicatedConfigMapSpec{ + ReplicatedResourceSpec: &api.ReplicatedResourceSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "configMap-controller": "test", + coreapi.K8ssandraClusterNamespaceLabel: namespace, + coreapi.K8ssandraClusterNameLabel: "k8sssandra", + }, + }, + ReplicationTargets: []api.ReplicationTarget{ + { + K8sContextName: k8sContext, + }, + }, + }, + }, + } +} diff --git a/controllers/replication/secret_controller.go b/controllers/replication/secret_controller.go index 3a158457e..135b31563 100644 --- a/controllers/replication/secret_controller.go +++ b/controllers/replication/secret_controller.go @@ -3,10 +3,11 @@ package replication import ( "context" "fmt" - "github.com/k8ssandra/k8ssandra-operator/pkg/secret" "strings" "sync" + "github.com/k8ssandra/k8ssandra-operator/pkg/secret" + coreapi "github.com/k8ssandra/k8ssandra-operator/apis/k8ssandra/v1alpha1" api "github.com/k8ssandra/k8ssandra-operator/apis/replication/v1alpha1" "github.com/k8ssandra/k8ssandra-operator/pkg/clientcache" @@ -249,7 +250,7 @@ func (s *SecretSyncController) Reconcile(ctx context.Context, req ctrl.Request) break TargetSecrets } - if requiresUpdate(sec, fetchedSecret) { + if objectRequiresUpdate(sec, fetchedSecret) { logger.Info("Modifying secret in target cluster", "Secret", sec.Name, "TargetContext", target) syncSecrets(sec, fetchedSecret) if err = remoteClient.Update(ctx, fetchedSecret); err != nil { @@ -285,59 +286,34 @@ func (s *SecretSyncController) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, err } -func requiresUpdate(source, dest client.Object) bool { - // In case we target the same cluster - if source.GetUID() == dest.GetUID() { - return false - } - - if srcHash, found := source.GetAnnotations()[coreapi.ResourceHashAnnotation]; found { - // Get dest hash value - destHash, destFound := dest.GetAnnotations()[coreapi.ResourceHashAnnotation] - if !destFound { - return true - } - - if destSec, valid := dest.(*corev1.Secret); valid { - hash := utils.DeepHashString(destSec.Data) - if destHash != hash { - // Destination data did not match destination hash - return true - } - } - - return srcHash != destHash - } - return false -} - func syncSecrets(src, dest *corev1.Secret) { origMeta := dest.ObjectMeta src.DeepCopyInto(dest) dest.ObjectMeta = origMeta dest.OwnerReferences = []metav1.OwnerReference{} - - // sync annotations, src is more important - if dest.GetAnnotations() == nil { - dest.Annotations = make(map[string]string) - } - - for k, v := range src.Annotations { - if !filterValue(k) { - dest.Annotations[k] = v - } - } - - // sync labels, src is more important - if dest.GetLabels() == nil { - dest.Labels = make(map[string]string) - } - - for k, v := range src.Labels { - if !filterValue(k) { - dest.Labels[k] = v - } - } + syncMetadata(&src.ObjectMeta, &dest.ObjectMeta) + + // // sync annotations, src is more important + // if dest.GetAnnotations() == nil { + // dest.Annotations = make(map[string]string) + // } + + // for k, v := range src.Annotations { + // if !filterValue(k) { + // dest.Annotations[k] = v + // } + // } + + // // sync labels, src is more important + // if dest.GetLabels() == nil { + // dest.Labels = make(map[string]string) + // } + + // for k, v := range src.Labels { + // if !filterValue(k) { + // dest.Labels[k] = v + // } + // } } // filterValue verifies the annotation is not something datacenter specific diff --git a/controllers/replication/secret_controller_test.go b/controllers/replication/secret_controller_test.go index ca4f996ec..43586eb98 100644 --- a/controllers/replication/secret_controller_test.go +++ b/controllers/replication/secret_controller_test.go @@ -3,6 +3,9 @@ package replication import ( "context" "fmt" + "testing" + "time" + "github.com/go-logr/logr" "github.com/k8ssandra/k8ssandra-operator/pkg/clientcache" "github.com/k8ssandra/k8ssandra-operator/pkg/config" @@ -10,8 +13,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/cluster" "sigs.k8s.io/controller-runtime/pkg/manager" - "testing" - "time" coreapi "github.com/k8ssandra/k8ssandra-operator/apis/k8ssandra/v1alpha1" api "github.com/k8ssandra/k8ssandra-operator/apis/replication/v1alpha1" @@ -298,16 +299,18 @@ func generateReplicatedSecret(k8sContext, namespace string) *api.ReplicatedSecre Name: "fetch-secrets", }, Spec: api.ReplicatedSecretSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "secret-controller": "test", - coreapi.K8ssandraClusterNamespaceLabel: namespace, - coreapi.K8ssandraClusterNameLabel: "k8sssandra", + ReplicatedResourceSpec: &api.ReplicatedResourceSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "secret-controller": "test", + coreapi.K8ssandraClusterNamespaceLabel: namespace, + coreapi.K8ssandraClusterNameLabel: "k8sssandra", + }, }, - }, - ReplicationTargets: []api.ReplicationTarget{ - { - K8sContextName: k8sContext, + ReplicationTargets: []api.ReplicationTarget{ + { + K8sContextName: k8sContext, + }, }, }, }, @@ -459,16 +462,16 @@ func TestRequiresUpdate(t *testing.T) { orig.GetAnnotations()[coreapi.ResourceHashAnnotation] = utils.DeepHashString(orig.Data) // Secrets don't match - assert.True(requiresUpdate(orig, dest)) + assert.True(objectRequiresUpdate(orig, dest)) syncSecrets(orig, dest) - assert.False(requiresUpdate(orig, dest)) + assert.False(objectRequiresUpdate(orig, dest)) // Modify target data without fixing the hash annotation, this should cause update requirement dest.Data["secondKey"] = []byte("thisValWillBeGone") - assert.True(requiresUpdate(orig, dest)) + assert.True(objectRequiresUpdate(orig, dest)) syncSecrets(orig, dest) - assert.False(requiresUpdate(orig, dest)) + assert.False(objectRequiresUpdate(orig, dest)) } diff --git a/pkg/encryption/zz_generated.deepcopy.go b/pkg/encryption/zz_generated.deepcopy.go index db1650155..47448fbb9 100644 --- a/pkg/encryption/zz_generated.deepcopy.go +++ b/pkg/encryption/zz_generated.deepcopy.go @@ -2,7 +2,7 @@ // +build !ignore_autogenerated /* -Copyright 2021. +Copyright 2022. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/images/zz_generated.deepcopy.go b/pkg/images/zz_generated.deepcopy.go index ba9c8d296..428702b72 100644 --- a/pkg/images/zz_generated.deepcopy.go +++ b/pkg/images/zz_generated.deepcopy.go @@ -2,7 +2,7 @@ // +build !ignore_autogenerated /* -Copyright 2021. +Copyright 2022. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/secret/replicated.go b/pkg/secret/replicated.go index 4d3a61d88..7686e2e5e 100644 --- a/pkg/secret/replicated.go +++ b/pkg/secret/replicated.go @@ -4,6 +4,9 @@ import ( "context" "crypto/rand" "fmt" + "math/big" + "reflect" + "github.com/go-logr/logr" "github.com/k8ssandra/k8ssandra-operator/pkg/annotations" "github.com/k8ssandra/k8ssandra-operator/pkg/labels" @@ -12,8 +15,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - "math/big" - "reflect" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -163,10 +164,12 @@ func generateReplicatedSecret(kcKey client.ObjectKey, replicationTargets []repli return &replicationapi.ReplicatedSecret{ ObjectMeta: getManagedObjectMeta(kcKey.Name, kcKey), Spec: replicationapi.ReplicatedSecretSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: labels.ManagedByLabels(kcKey), + ReplicatedResourceSpec: &replicationapi.ReplicatedResourceSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: labels.ManagedByLabels(kcKey), + }, + ReplicationTargets: replicationTargets, }, - ReplicationTargets: replicationTargets, }, } }