diff --git a/apis/offloading/v1beta1/virtualnode_types.go b/apis/offloading/v1beta1/virtualnode_types.go index c2334a0d93..c8bd3b8f52 100644 --- a/apis/offloading/v1beta1/virtualnode_types.go +++ b/apis/offloading/v1beta1/virtualnode_types.go @@ -39,6 +39,8 @@ type OffloadingPatch struct { Tolerations []corev1.Toleration `json:"tolerations,omitempty"` // Affinity contains the affinity and anti-affinity rules to target the remote cluster. Affinity *Affinity `json:"affinity,omitempty"` + // RuntimeClassName contains the runtimeclass name the pods should have on the target remote cluster. + RuntimeClassName *string `json:"runtimeClassName,omitempty"` } // DeploymentTemplate contains the deployment template of the virtual node. diff --git a/apis/offloading/v1beta1/zz_generated.deepcopy.go b/apis/offloading/v1beta1/zz_generated.deepcopy.go index 2a2e5f85c9..90c88928f0 100644 --- a/apis/offloading/v1beta1/zz_generated.deepcopy.go +++ b/apis/offloading/v1beta1/zz_generated.deepcopy.go @@ -334,6 +334,11 @@ func (in *OffloadingPatch) DeepCopyInto(out *OffloadingPatch) { *out = new(Affinity) (*in).DeepCopyInto(*out) } + if in.RuntimeClassName != nil { + in, out := &in.RuntimeClassName, &out.RuntimeClassName + *out = new(string) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OffloadingPatch. diff --git a/cmd/webhook/main.go b/cmd/webhook/main.go index 24d2f0413e..90fdefe521 100644 --- a/cmd/webhook/main.go +++ b/cmd/webhook/main.go @@ -90,7 +90,7 @@ func main() { "Enforce offerer-side that offloaded pods do not exceed offered resources (based on container limits)") refreshInterval := pflag.Duration("resource-validator-refresh-interval", 5*time.Minute, "The interval at which the resource validator cache is refreshed") - liqoRuntimeClassName := pflag.String("liqo-runtime-class", "liqo", + liqoRuntimeClassName := pflag.String("liqo-runtime-class", consts.LiqoRuntimeClassName, "Define the Liqo runtime class forcing the pods to be scheduled on virtual nodes") flagsutils.InitKlogFlags(pflag.CommandLine) diff --git a/deployments/liqo/charts/liqo-crds/crds/offloading.liqo.io_virtualnodes.yaml b/deployments/liqo/charts/liqo-crds/crds/offloading.liqo.io_virtualnodes.yaml index 0954308d15..1c39cfefd6 100644 --- a/deployments/liqo/charts/liqo-crds/crds/offloading.liqo.io_virtualnodes.yaml +++ b/deployments/liqo/charts/liqo-crds/crds/offloading.liqo.io_virtualnodes.yaml @@ -389,6 +389,10 @@ spec: description: NodeSelector contains the node selector to target the remote cluster. type: object + runtimeClassName: + description: RuntimeClassName contains the runtimeclass name the + pods should have on the target remote cluster. + type: string tolerations: description: Tolerations contains the tolerations to target the remote cluster. diff --git a/docs/usage/reflection.md b/docs/usage/reflection.md index 9d3146a48c..de03883d34 100644 --- a/docs/usage/reflection.md +++ b/docs/usage/reflection.md @@ -192,3 +192,21 @@ More specifically, an event is propagated if it belongs to an offloaded namespac The event reflector is the only one that propagates a resource from the remote cluster to the local cluster. Local events are not reflected to the remote cluster. ``` + +(UsageReflectionRuntimeClass)= + +## RuntimeClass + +The **RuntimeClass** (`.spec.runtimeClassName` field) is reflected from the local pod to the remote one. + +If you are using the [Liqo RuntimeClass](../usage/namespace-offloading.md#runtimeclass), you cannot specify the RuntimeClass name as the field is already used. +To overcome this problem, you can annotate the pod with `liqo.io/remote-runtime-class-name: `. + +It is also possible to enforce a remote RuntimeClass to all pods scheduled on a virtual node, by specifying it in the *OffladingPatch* of the virtualnode (`.spec.offloadingPatch.runtimeClassName`). +If you are using liqoctl to create the virtual node, you can leverage the `--runtime-class-name` flag. + +If these options are used in combination, the following priority (from higher to lower priority) will be used to determine the remote RuntimeClass: + +1. pod Annotation (`liqo.io/remote-runtime-class-name`). +2. pod RuntimeClass (`.spec.runtimeClassName`). It is ignored if set to `liqo`. +3. virtualNode OffloadingPatch (`.spec.offloadingPatch.runtimeClassName`). diff --git a/pkg/consts/core.go b/pkg/consts/core.go index e5f83e1f78..0c6cf71143 100644 --- a/pkg/consts/core.go +++ b/pkg/consts/core.go @@ -23,4 +23,7 @@ const ( // LiqoAppLabelValue is the value of the label used to identify Liqo app. LiqoAppLabelValue = "liqo" + + // LiqoRuntimeClassName is the name of the runtimeclass used by Liqo. + LiqoRuntimeClassName = "liqo" ) diff --git a/pkg/consts/replication.go b/pkg/consts/replication.go index 76357a0bf9..0305a6dcab 100644 --- a/pkg/consts/replication.go +++ b/pkg/consts/replication.go @@ -84,4 +84,7 @@ const ( // RemoteUnavailableValue is the annotation value of the label indicating that the remote cluster hosting the local offloaded pod is currently // not available. RemoteUnavailableValue = "true" + + // RemoteRuntimeClassNameAnnotKey is the annotation key used to store the name of the remote pod runtimeclass. + RemoteRuntimeClassNameAnnotKey = "liqo.io/remote-runtime-class-name" ) diff --git a/pkg/liqo-controller-manager/offloading/forge/virtualnode.go b/pkg/liqo-controller-manager/offloading/forge/virtualnode.go index e119adcbbe..9b9d500bb5 100644 --- a/pkg/liqo-controller-manager/offloading/forge/virtualnode.go +++ b/pkg/liqo-controller-manager/offloading/forge/virtualnode.go @@ -58,7 +58,7 @@ func VirtualNode(name, namespace string) *offloadingv1beta1.VirtualNode { // MutateVirtualNode mutates a VirtualNode resource. func MutateVirtualNode(ctx context.Context, cl client.Client, virtualNode *offloadingv1beta1.VirtualNode, - remoteClusterID liqov1beta1.ClusterID, opts *VirtualNodeOptions, createNode, disableNetworkCheck *bool) error { + remoteClusterID liqov1beta1.ClusterID, opts *VirtualNodeOptions, createNode, disableNetworkCheck *bool, runtimeClassName *string) error { // VirtualNode metadata if virtualNode.ObjectMeta.Labels == nil { virtualNode.ObjectMeta.Labels = make(map[string]string) @@ -89,6 +89,13 @@ func MutateVirtualNode(ctx context.Context, cl client.Client, virtualNode *offlo virtualNode.Spec.IngressClasses = opts.IngressClasses virtualNode.Spec.LoadBalancerClasses = opts.LoadBalancerClasses + if runtimeClassName != nil && *runtimeClassName != "" { + if virtualNode.Spec.OffloadingPatch == nil { + virtualNode.Spec.OffloadingPatch = &offloadingv1beta1.OffloadingPatch{} + } + virtualNode.Spec.OffloadingPatch.RuntimeClassName = runtimeClassName + } + if len(opts.NodeSelector) > 0 { if virtualNode.Spec.OffloadingPatch == nil { virtualNode.Spec.OffloadingPatch = &offloadingv1beta1.OffloadingPatch{} diff --git a/pkg/liqo-controller-manager/virtualnodecreator-controller/virtualnodecreator_controller.go b/pkg/liqo-controller-manager/virtualnodecreator-controller/virtualnodecreator_controller.go index 032495fd00..8bce812cc5 100644 --- a/pkg/liqo-controller-manager/virtualnodecreator-controller/virtualnodecreator_controller.go +++ b/pkg/liqo-controller-manager/virtualnodecreator-controller/virtualnodecreator_controller.go @@ -125,7 +125,7 @@ func (r *VirtualNodeCreatorReconciler) Reconcile(ctx context.Context, req ctrl.R virtualNode.Spec.VkOptionsTemplateRef) if err := forge.MutateVirtualNode(ctx, r.Client, - virtualNode, identity.Spec.ClusterID, vnOpts, nil, nil); err != nil { + virtualNode, identity.Spec.ClusterID, vnOpts, nil, nil, nil); err != nil { return err } if virtualNode.Labels == nil { diff --git a/pkg/liqoctl/rest/virtualnode/create.go b/pkg/liqoctl/rest/virtualnode/create.go index 80931a2d27..6e030176d6 100644 --- a/pkg/liqoctl/rest/virtualnode/create.go +++ b/pkg/liqoctl/rest/virtualnode/create.go @@ -102,6 +102,7 @@ func (o *Options) Create(ctx context.Context, options *rest.CreateOptions) *cobr []string{}, "The load balancer classes offered by the remote cluster. The first one will be used as default") cmd.Flags().StringToStringVar(&o.labels, "labels", map[string]string{}, "The labels to be added to the virtual node") cmd.Flags().StringToStringVar(&o.nodeSelector, "node-selector", map[string]string{}, "The node selector to be applied to offloaded pods") + cmd.Flags().StringVar(&o.runtimeClassName, "runtime-class-name", "", "The runtimeClass the pods should have on the target remote cluster") runtime.Must(cmd.MarkFlagRequired("remote-cluster-id")) @@ -166,7 +167,7 @@ func (o *Options) handleCreate(ctx context.Context) error { virtualNode := forge.VirtualNode(opts.Name, tenantNamespace) if _, err := resource.CreateOrUpdate(ctx, opts.CRClient, virtualNode, func() error { return forge.MutateVirtualNode(ctx, opts.CRClient, - virtualNode, o.remoteClusterID.GetClusterID(), vnOpts, &o.createNode, &o.disableNetworkCheck) + virtualNode, o.remoteClusterID.GetClusterID(), vnOpts, &o.createNode, &o.disableNetworkCheck, &o.runtimeClassName) }); err != nil { s.Fail("Unable to create virtual node: ", output.PrettyErr(err)) return err @@ -301,7 +302,7 @@ func (o *Options) output(ctx context.Context, name, namespace string, vnOpts *fo virtualNode := forge.VirtualNode(name, namespace) if err := forge.MutateVirtualNode(ctx, opts.CRClient, - virtualNode, o.remoteClusterID.GetClusterID(), vnOpts, &o.createNode, &o.disableNetworkCheck); err != nil { + virtualNode, o.remoteClusterID.GetClusterID(), vnOpts, &o.createNode, &o.disableNetworkCheck, &o.runtimeClassName); err != nil { return err } diff --git a/pkg/liqoctl/rest/virtualnode/types.go b/pkg/liqoctl/rest/virtualnode/types.go index d55ae16e2a..3f709a41e3 100644 --- a/pkg/liqoctl/rest/virtualnode/types.go +++ b/pkg/liqoctl/rest/virtualnode/types.go @@ -42,6 +42,7 @@ type Options struct { loadBalancerClasses []string labels map[string]string nodeSelector map[string]string + runtimeClassName string } var _ rest.API = &Options{} diff --git a/pkg/virtualKubelet/forge/forge.go b/pkg/virtualKubelet/forge/forge.go index f382f46027..3d09e290f3 100644 --- a/pkg/virtualKubelet/forge/forge.go +++ b/pkg/virtualKubelet/forge/forge.go @@ -77,6 +77,7 @@ type ForgingOpts struct { NodeSelector map[string]string Tolerations []corev1.Toleration Affinity *offloadingv1beta1.Affinity + RuntimeClassName *string } // NewForgingOpts returns a new ForgingOpts instance. @@ -91,6 +92,7 @@ func NewForgingOpts(offloadingPatch *offloadingv1beta1.OffloadingPatch) ForgingO NodeSelector: offloadingPatch.NodeSelector, Tolerations: offloadingPatch.Tolerations, Affinity: offloadingPatch.Affinity, + RuntimeClassName: offloadingPatch.RuntimeClassName, } } diff --git a/pkg/virtualKubelet/forge/pods.go b/pkg/virtualKubelet/forge/pods.go index c9daecf778..e56ff58b38 100644 --- a/pkg/virtualKubelet/forge/pods.go +++ b/pkg/virtualKubelet/forge/pods.go @@ -182,6 +182,8 @@ func RemoteShadowPod(local *corev1.Pod, remote *offloadingv1beta1.ShadowPod, AntiAffinityHardMutator(FilterAntiAffinityLabels(localMetaFiltered.GetLabels(), local.Annotations[liqoconst.PodAntiAffinityLabelsKey]))) } + mutators = append(mutators, RuntimeClassNameMutator(local, forgingOpts)) + return &offloadingv1beta1.ShadowPod{ ObjectMeta: RemoteObjectMeta(localMetaFiltered, &remote.ObjectMeta), Spec: offloadingv1beta1.ShadowPodSpec{ @@ -421,6 +423,30 @@ func FilterAntiAffinityLabels(labels map[string]string, whitelist string) map[st appsv1.DefaultDeploymentUniqueLabelKey, appsv1.StatefulSetPodNameLabel)) } +// RuntimeClassNameMutator is a mutator which implements the support to propagate the runtimeclass name. +func RuntimeClassNameMutator(local *corev1.Pod, forgingOpts *ForgingOpts) RemotePodSpecMutator { + return func(remote *corev1.PodSpec) { + // 1st priority: use RuntimeClass from pod annotation if set. + if v, ok := local.GetAnnotations()[liqoconst.RemoteRuntimeClassNameAnnotKey]; ok && v != "" { + remote.RuntimeClassName = &v + return + } + + // 2nd priority: use RuntimeClass from local pod spec if set (and not equal to "liqo"). + if local.Spec.RuntimeClassName != nil && + *local.Spec.RuntimeClassName != "" && *local.Spec.RuntimeClassName != liqoconst.LiqoRuntimeClassName { + remote.RuntimeClassName = local.Spec.RuntimeClassName + return + } + + // 3rd priority: use RuntimeClass from virtualnode OffloadingPatch if set. + if forgingOpts != nil && forgingOpts.RuntimeClassName != nil && + *forgingOpts.RuntimeClassName != "" && *forgingOpts.RuntimeClassName != liqoconst.LiqoRuntimeClassName { + remote.RuntimeClassName = forgingOpts.RuntimeClassName + } + } +} + // RemoteContainersAPIServerSupport forges the containers for a reflected pod, appropriately adding the environment variables // to enable the offloaded containers to contact back the local API server, instead of the remote one. func RemoteContainersAPIServerSupport(containers []corev1.Container, saName, homeAPIServerHost, homeAPIServerPort string) []corev1.Container { diff --git a/pkg/virtualKubelet/forge/pods_test.go b/pkg/virtualKubelet/forge/pods_test.go index 04bf544e1f..71e6c3c7ed 100644 --- a/pkg/virtualKubelet/forge/pods_test.go +++ b/pkg/virtualKubelet/forge/pods_test.go @@ -28,6 +28,7 @@ import ( corev1apply "k8s.io/client-go/applyconfigurations/core/v1" metricsv1beta1 "k8s.io/metrics/pkg/apis/metrics/v1beta1" "k8s.io/utils/pointer" + "k8s.io/utils/ptr" offloadingv1beta1 "github.com/liqotech/liqo/apis/offloading/v1beta1" "github.com/liqotech/liqo/pkg/consts" @@ -458,6 +459,99 @@ var _ = Describe("Pod forging", func() { }) }) + Describe("the runtimeClassNameMutator function", func() { + const ( + fakeRuntimeClassOffPatch = "foo" + fakeRuntimeClassPodSpec = "bar" + fakeRuntimeClassPodAnnot = "baz" + ) + + var ( + local *corev1.Pod + remote *corev1.PodSpec + forgingOpts *forge.ForgingOpts + ) + + BeforeEach(func() { + local = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "bar", + }, + Spec: corev1.PodSpec{}, + } + remote = &corev1.PodSpec{} + forgingOpts = &forge.ForgingOpts{} + }) + + JustBeforeEach(func() { forge.RuntimeClassNameMutator(local, forgingOpts)(remote) }) + + When("runtimeclass is set in the OffloadingPatch", func() { + BeforeEach(func() { + forgingOpts.RuntimeClassName = ptr.To(fakeRuntimeClassOffPatch) + }) + + It("should set the runtimeclass of the OffloadingPatch", func() { + Expect(remote.RuntimeClassName).To(PointTo(Equal(fakeRuntimeClassOffPatch))) + }) + }) + + When("runtimeclass is set in the OffloadingPatch and in the Pod spec", func() { + BeforeEach(func() { + forgingOpts.RuntimeClassName = ptr.To(fakeRuntimeClassOffPatch) + local.Spec.RuntimeClassName = ptr.To(fakeRuntimeClassPodSpec) + }) + + It("should set the runtimeclass of the pod spec", func() { + Expect(remote.RuntimeClassName).To(PointTo(Equal(fakeRuntimeClassPodSpec))) + }) + }) + + When("runtimeclass is set in the OffloadingPatch, Pod spec and Pod annotation", func() { + BeforeEach(func() { + forgingOpts.RuntimeClassName = ptr.To(fakeRuntimeClassOffPatch) + local.Spec.RuntimeClassName = ptr.To(fakeRuntimeClassPodSpec) + local.Annotations = map[string]string{consts.RemoteRuntimeClassNameAnnotKey: fakeRuntimeClassPodAnnot} + }) + + It("should set the runtimeclass of the pod annotation", func() { + Expect(remote.RuntimeClassName).To(PointTo(Equal(fakeRuntimeClassPodAnnot))) + }) + }) + + When("runtimeclass is set to the liqo one", func() { + BeforeEach(func() { + local.Spec.RuntimeClassName = ptr.To(consts.LiqoRuntimeClassName) + }) + + When("OffloadingPatch and Pod annotation are not set", func() { + It("should leave the runtimeclass empty", func() { + Expect(remote.RuntimeClassName).To(BeNil()) + }) + }) + + When("OffloadingPatch is set", func() { + BeforeEach(func() { + forgingOpts.RuntimeClassName = ptr.To(fakeRuntimeClassOffPatch) + }) + + It("should set the runtimeclass of the OffloadingPatch", func() { + Expect(remote.RuntimeClassName).To(PointTo(Equal(fakeRuntimeClassOffPatch))) + }) + }) + + When("Pod annotation is set", func() { + BeforeEach(func() { + local.Annotations = map[string]string{consts.RemoteRuntimeClassNameAnnotKey: fakeRuntimeClassPodAnnot} + }) + + It("should set the runtimeclass of the Pod annotation", func() { + Expect(remote.RuntimeClassName).To(PointTo(Equal(fakeRuntimeClassPodAnnot))) + }) + }) + }) + }) + Describe("the RemoteContainersAPIServerSupport function", func() { var container corev1.Container var output []corev1.Container