Merge pull request #1155 from Tal-or/serial_suite_hcp

CNF-16537:Adjust serial suite for running on HyperShift platform [tier0]

internal/controller/kubeletconfig_controller.go

@@ -27,7 +27,6 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
-	serializer "k8s.io/apimachinery/pkg/runtime/serializer/json"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/tools/record"
 	"k8s.io/klog/v2"
@@ -270,8 +269,7 @@ func (r *KubeletConfigReconciler) makeKCHandlerForPlatform(ctx context.Context,
 
 		nodePoolName := cmKc.Labels[HyperShiftNodePoolLabel]
 		kcData := cmKc.Data[HyperShiftConfigMapConfigKey]
-		mcoKc := &mcov1.KubeletConfig{}
-		err := decodeKCFrom([]byte(kcData), r.Scheme, mcoKc)
+		mcoKc, err := kubeletconfig.DecodeFromData([]byte(kcData), r.Scheme)
 		if err != nil {
 			return nil, err
 		}
@@ -339,12 +337,3 @@ func removeDeletedOwner(kcKey client.ObjectKey, ownerConfigMaps []*corev1.Config
 	}
 	return ownerConfigMaps
 }
-
-func decodeKCFrom(data []byte, scheme *runtime.Scheme, mcoKc *mcov1.KubeletConfig) error {
-	yamlSerializer := serializer.NewSerializerWithOptions(
-		serializer.DefaultMetaFactory, scheme, scheme,
-		serializer.SerializerOptions{Yaml: true, Pretty: true, Strict: true})
-
-	_, _, err := yamlSerializer.Decode(data, nil, mcoKc)
-	return err
-}

internal/hypershift/consts/consts.go

@@ -0,0 +1,14 @@
+package consts
+
+const (
+	// NodePoolNameLabel uses to label ConfigMap objects which are associated with the NodePool
+	NodePoolNameLabel = "hypershift.openshift.io/nodePool"
+
+	// KubeletConfigConfigMapLabel uses
+	// to label a ConfigMap that holds a KubeletConfig object
+	KubeletConfigConfigMapLabel = "hypershift.openshift.io/kubeletconfig-config"
+
+	// ConfigKey is the key under ConfigMap.Data on which encoded
+	// machine-config, kubelet-config objects are stored.
+	ConfigKey = "config"
+)

internal/machineconfigpools/machineconfigpools.go

@@ -19,6 +19,7 @@ package machineconfigpools
 import (
 	"context"
 
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -67,3 +68,18 @@ func FindBySelector(mcps []*mcov1.MachineConfigPool, sel *metav1.LabelSelector)
 	}
 	return nil, &NotFound{Selector: sel.String()}
 }
+
+func GetNodesFrom(ctx context.Context, cli client.Client, mcp *mcov1.MachineConfigPool) ([]corev1.Node, error) {
+	sel, err := metav1.LabelSelectorAsSelector(mcp.Spec.NodeSelector)
+	if err != nil {
+		return nil, err
+	}
+
+	nodeList := &corev1.NodeList{}
+	err = cli.List(ctx, nodeList, &client.ListOptions{LabelSelector: sel})
+	if err != nil {
+		return nil, err
+	}
+
+	return nodeList.Items, nil
+}

internal/nodegroups/nodegroups.go

@@ -0,0 +1,112 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright 2025 Red Hat, Inc.
+ */
+
+package nodegroups
+
+import (
+	"context"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/klog/v2"
+
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/k8stopologyawareschedwg/deployer/pkg/deployer/platform"
+	"github.com/k8stopologyawareschedwg/deployer/pkg/deployer/platform/detect"
+
+	nropv1 "github.com/openshift-kni/numaresources-operator/api/v1"
+	"github.com/openshift-kni/numaresources-operator/internal/hypershift/consts"
+	"github.com/openshift-kni/numaresources-operator/internal/machineconfigpools"
+	mcov1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1"
+)
+
+func GetNodesFrom(ctx context.Context, cli client.Client, nodeGroups []nropv1.NodeGroup) ([]corev1.Node, error) {
+	plat, err := detect.Platform(ctx)
+	if err != nil {
+		return nil, err
+	}
+	nodes := make([]corev1.Node, 0)
+	if plat == platform.HyperShift {
+		nodeList := corev1.NodeList{}
+		for _, nodeGroup := range nodeGroups {
+			opts := &client.ListOptions{
+				LabelSelector: labels.SelectorFromSet(map[string]string{
+					consts.NodePoolNameLabel: *nodeGroup.PoolName,
+				}),
+			}
+			if err := cli.List(ctx, &nodeList, opts); err != nil {
+				return nodes, err
+			}
+			nodes = append(nodes, nodeList.Items...)
+		}
+		return nodes, nil
+	}
+
+	nroMcps, err := machineconfigpools.GetListByNodeGroupsV1(ctx, cli, nodeGroups)
+	if err != nil {
+		return nodes, err
+	}
+	for _, mcp := range nroMcps {
+		mcpNodes, err := machineconfigpools.GetNodesFrom(ctx, cli, mcp)
+		if err != nil {
+			return nodes, err
+		}
+		nodes = append(nodes, mcpNodes...)
+	}
+	return nodes, nil
+}
+
+func GetPoolNamesFrom(ctx context.Context, cli client.Client, nodeGroups []nropv1.NodeGroup) ([]string, error) {
+	poolNames := make([]string, 0)
+	for _, nodeGroup := range nodeGroups {
+		// this should cover HyperShift cases and OpenShift cases when PoolName in use
+		if *nodeGroup.PoolName != "" {
+			poolNames = append(poolNames, *nodeGroup.PoolName)
+			continue
+		}
+		// fallback to MCPs if PoolName is not in use
+		mcps, err := machineconfigpools.GetListByNodeGroupsV1(ctx, cli, []nropv1.NodeGroup{nodeGroup})
+		if err != nil {
+			return poolNames, err
+		}
+		for _, mcp := range mcps {
+			if mcp.Spec.NodeSelector == nil {
+				klog.Warningf("the machine config pool %q does not have node selector", mcp.Name)
+				continue
+			}
+			poolNames = append(poolNames, mcp.Name)
+		}
+	}
+	return poolNames, nil
+}
+
+func NodeSelectorFromPoolName(ctx context.Context, cli client.Client, poolName string) (map[string]string, error) {
+	plat, err := detect.Platform(ctx)
+	if err != nil {
+		return nil, err
+	}
+	if plat == platform.HyperShift {
+		return map[string]string{
+			consts.NodePoolNameLabel: poolName,
+		}, nil
+	}
+	mcp := &mcov1.MachineConfigPool{}
+	err = cli.Get(ctx, client.ObjectKey{Name: poolName}, mcp)
+	if err != nil {
+		return nil, err
+	}
+	return mcp.Spec.NodeSelector.MatchLabels, nil
+}

nrovalidate/validator/kubeletconfig.go

@@ -20,11 +20,16 @@ import (
 	"context"
 	"fmt"
 
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/klog/v2"
 	kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
+	"github.com/k8stopologyawareschedwg/deployer/pkg/deployer/platform"
+	"github.com/k8stopologyawareschedwg/deployer/pkg/deployer/platform/detect"
 	deployervalidator "github.com/k8stopologyawareschedwg/deployer/pkg/validator"
+	hypershiftconsts "github.com/openshift-kni/numaresources-operator/internal/hypershift/consts"
 	"github.com/openshift-kni/numaresources-operator/pkg/kubeletconfig"
 	mcov1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1"
 )
@@ -33,7 +38,59 @@ const (
 	ValidatorKubeletConfig = "k8scfg"
 )
 
-func CollectKubeletConfig(ctx context.Context, cli client.Client, data *ValidatorData) error {
+func CollectKubeletConfigForHyperShift(ctx context.Context, cli client.Client, data *ValidatorData) error {
+	cmList := &corev1.ConfigMapList{}
+	opts := []client.ListOption{
+		client.MatchingLabels(map[string]string{
+			hypershiftconsts.KubeletConfigConfigMapLabel: "true",
+		}),
+	}
+	if err := cli.List(ctx, cmList, opts...); err != nil {
+		return err
+	}
+
+	kConfigs := make(map[string]*kubeletconfigv1beta1.KubeletConfiguration)
+	for _, cm := range cmList.Items {
+		v, ok := cm.Data[hypershiftconsts.ConfigKey]
+		namespacedName := client.ObjectKeyFromObject(&cm).String()
+		if !ok {
+			klog.InfoS("Skipping KubeletConfig ConfigMap, Config key in data is missing", "ConfigMap", namespacedName)
+			continue
+		}
+
+		nodePoolName, ok := cm.Labels[hypershiftconsts.NodePoolNameLabel]
+		if !ok {
+			klog.InfoS("Skipping KubeletConfig ConfigMap, NodePool name is missing", "ConfigMap", namespacedName)
+			continue
+		}
+
+		mcoKc, err := kubeletconfig.DecodeFromData([]byte(v), cli.Scheme())
+		if err != nil {
+			return err
+		}
+
+		kubeletConfig, err := kubeletconfig.MCOKubeletConfToKubeletConf(mcoKc)
+		if err != nil {
+			return err
+		}
+		nodes := &corev1.NodeList{}
+		opts = []client.ListOption{
+			client.MatchingLabels(map[string]string{
+				hypershiftconsts.NodePoolNameLabel: nodePoolName,
+			}),
+		}
+		if err := cli.List(ctx, nodes, opts...); err != nil {
+			return err
+		}
+		if err := updateNodesForValidatorData(nodes.Items, kConfigs, kubeletConfig, data); err != nil {
+			return err
+		}
+	}
+	data.kConfigs = kConfigs
+	return nil
+}
+
+func CollectKubeletConfigForOpenShift(ctx context.Context, cli client.Client, data *ValidatorData) error {
 	mcoKubeletConfigList := mcov1.KubeletConfigList{}
 	if err := cli.List(ctx, &mcoKubeletConfigList); err != nil {
 		return err
@@ -55,19 +112,8 @@ func CollectKubeletConfig(ctx context.Context, cli client.Client, data *Validato
 			if err != nil {
 				return err
 			}
-			nodeNames := getNodeNames(nodes)
-
-			for _, nodeName := range nodeNames {
-				// we are just interested on nodes with TAS enabled
-				if !data.tasEnabledNodeNames.Has(nodeName) {
-					continue
-				}
-
-				_, found := data.kConfigs[nodeName]
-				if found {
-					return fmt.Errorf("Found two KubeletConfigurations for node %q", nodeName)
-				}
-				kConfigs[nodeName] = kubeletConfig
+			if err := updateNodesForValidatorData(nodes, kConfigs, kubeletConfig, data); err != nil {
+				return err
 			}
 		}
 	}
@@ -76,6 +122,17 @@ func CollectKubeletConfig(ctx context.Context, cli client.Client, data *Validato
 	return nil
 }
 
+func CollectKubeletConfig(ctx context.Context, cli client.Client, data *ValidatorData) error {
+	plat, err := detect.Platform(ctx)
+	if err != nil {
+		return err
+	}
+	if plat == platform.HyperShift {
+		return CollectKubeletConfigForHyperShift(ctx, cli, data)
+	}
+	return CollectKubeletConfigForOpenShift(ctx, cli, data)
+}
+
 func ValidateKubeletConfig(data ValidatorData) ([]deployervalidator.ValidationResult, error) {
 	var ret []deployervalidator.ValidationResult
 	for _, nodeName := range sets.List(data.tasEnabledNodeNames) {
@@ -86,3 +143,26 @@ func ValidateKubeletConfig(data ValidatorData) ([]deployervalidator.ValidationRe
 	}
 	return ret, nil
 }
+
+func updateNodesForValidatorData(
+	nodes []corev1.Node,
+	kConfigs map[string]*kubeletconfigv1beta1.KubeletConfiguration,
+	kubeletConfig *kubeletconfigv1beta1.KubeletConfiguration,
+	data *ValidatorData,
+) error {
+	nodeNames := getNodeNames(nodes)
+
+	for _, nodeName := range nodeNames {
+		// we are just interested on nodes with TAS enabled
+		if !data.tasEnabledNodeNames.Has(nodeName) {
+			continue
+		}
+
+		_, found := data.kConfigs[nodeName]
+		if found {
+			return fmt.Errorf("found two KubeletConfigurations for node %q", nodeName)
+		}
+		kConfigs[nodeName] = kubeletConfig
+	}
+	return nil
+}

nrovalidate/validator/validator.go

@@ -34,11 +34,10 @@ import (
 	"github.com/k8stopologyawareschedwg/deployer/pkg/validator"
 	deployervalidator "github.com/k8stopologyawareschedwg/deployer/pkg/validator"
 
+	nrtv1alpha2 "github.com/k8stopologyawareschedwg/noderesourcetopology-api/pkg/apis/topology/v1alpha2"
 	nropv1 "github.com/openshift-kni/numaresources-operator/api/v1"
-	"github.com/openshift-kni/numaresources-operator/internal/machineconfigpools"
+	"github.com/openshift-kni/numaresources-operator/internal/nodegroups"
 	"github.com/openshift-kni/numaresources-operator/pkg/objectnames"
-
-	nrtv1alpha2 "github.com/k8stopologyawareschedwg/noderesourcetopology-api/pkg/apis/topology/v1alpha2"
 )
 
 type Report struct {
@@ -176,19 +175,11 @@ func GetNodesByNRO(ctx context.Context, cli client.Client) (sets.Set[string], er
 	if err != nil {
 		return enabledNodeNames, err
 	}
-
-	nroMcps, err := machineconfigpools.GetListByNodeGroupsV1(ctx, cli, nroInstance.Spec.NodeGroups)
+	nodes, err := nodegroups.GetNodesFrom(ctx, cli, nroInstance.Spec.NodeGroups)
 	if err != nil {
 		return enabledNodeNames, err
 	}
-
-	for _, mcp := range nroMcps {
-		nodes, err := getNodeListFromMachineConfigPool(ctx, cli, *mcp)
-		if err != nil {
-			return enabledNodeNames, err
-		}
-		enabledNodeNames.Insert(getNodeNames(nodes)...)
-	}
+	enabledNodeNames.Insert(getNodeNames(nodes)...)
 
 	return enabledNodeNames, nil
 }

pkg/kubeletconfig/kubeletconfig.go

@@ -20,6 +20,9 @@ import (
 	"encoding/json"
 	"errors"
 
+	"k8s.io/apimachinery/pkg/runtime"
+	serializer "k8s.io/apimachinery/pkg/runtime/serializer/json"
+
 	kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1"
 
 	mcov1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1"
@@ -46,3 +49,13 @@ func KubeletConfToMCKubeletConf(kcObj *kubeletconfigv1beta1.KubeletConfiguration
 	kcAsMc.Spec.KubeletConfig.Raw = rawKc
 	return err
 }
+
+func DecodeFromData(data []byte, scheme *runtime.Scheme) (*mcov1.KubeletConfig, error) {
+	mcoKc := &mcov1.KubeletConfig{}
+	yamlSerializer := serializer.NewSerializerWithOptions(
+		serializer.DefaultMetaFactory, scheme, scheme,
+		serializer.SerializerOptions{Yaml: true, Pretty: true, Strict: true})
+
+	_, _, err := yamlSerializer.Decode(data, nil, mcoKc)
+	return mcoKc, err
+}