Merge branch 'master' into sanprabhu/windows0_0_51

e2e/cluster.go

@@ -7,6 +7,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"strings"
 	"sync"
 	"testing"
 	"time"
@@ -17,7 +18,11 @@ import (
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v6"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources"
+	"github.com/stretchr/testify/require"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/tools/clientcmd"
 )
 
 var (
@@ -34,6 +39,12 @@ var (
 	clusterAzureNetworkOnce  sync.Once
 )
 
+type ClusterParams struct {
+	CACert         []byte
+	BootstrapToken string
+	FQDN           string
+}
+
 type Cluster struct {
 	Model         *armcontainerservice.ManagedCluster
 	Kube          *Kubeclient
@@ -127,15 +138,53 @@ func prepareCluster(ctx context.Context, t *testing.T, cluster *armcontainerserv
 		return nil, fmt.Errorf("collect garbage vmss: %w", err)
 	}
 
+	clusterParams, err := extractClusterParameters(ctx, t, kube, cluster)
+	if err != nil {
+		return nil, fmt.Errorf("extracting cluster parameters: %w", err)
+	}
+
 	return &Cluster{
 		Model:         cluster,
 		Kube:          kube,
 		SubnetID:      subnetID,
 		Maintenance:   maintenance,
-		ClusterParams: extractClusterParameters(ctx, t, kube),
+		ClusterParams: clusterParams,
 	}, nil
 }
 
+func extractClusterParameters(ctx context.Context, t *testing.T, kube *Kubeclient, cluster *armcontainerservice.ManagedCluster) (*ClusterParams, error) {
+	kubeconfig, err := clientcmd.Load(kube.KubeConfig)
+	if err != nil {
+		return nil, fmt.Errorf("loading cluster kubeconfig: %w", err)
+	}
+	clusterConfig := kubeconfig.Clusters[*cluster.Name]
+	if clusterConfig == nil {
+		return nil, fmt.Errorf("cluster kubeconfig missing configuration for %s", *cluster.Name)
+	}
+	return &ClusterParams{
+		CACert:         clusterConfig.CertificateAuthorityData,
+		BootstrapToken: getBootstrapToken(ctx, t, kube),
+		FQDN:           *cluster.Properties.Fqdn,
+	}, nil
+}
+
+func getBootstrapToken(ctx context.Context, t *testing.T, kube *Kubeclient) string {
+	secrets, err := kube.Typed.CoreV1().Secrets("kube-system").List(ctx, metav1.ListOptions{})
+	require.NoError(t, err)
+	secret := func() *corev1.Secret {
+		for _, secret := range secrets.Items {
+			if strings.HasPrefix(secret.Name, "bootstrap-token-") {
+				return &secret
+			}
+		}
+		t.Fatal("could not find secret with bootstrap-token- prefix")
+		return nil
+	}()
+	id := secret.Data["token-id"]
+	token := secret.Data["token-secret"]
+	return fmt.Sprintf("%s.%s", id, token)
+}
+
 func hash(cluster *armcontainerservice.ManagedCluster) string {
 	jsonData, err := json.Marshal(cluster)
 	if err != nil {

e2e/exec.go

@@ -5,13 +5,10 @@ import (
 	"context"
 	"fmt"
 	"strings"
-	"testing"
 
 	"github.com/Azure/agentbaker/e2e/config"
 	"github.com/google/uuid"
-	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/tools/remotecommand"
 )
@@ -32,66 +29,6 @@ func (r podExecResult) String() string {
 `, r.exitCode, r.stderr.String(), r.stdout.String())
 }
 
-type ClusterParams struct {
-	CACert         []byte
-	BootstrapToken string
-	FQDN           string
-	APIServerCert  []byte
-	ClientKey      []byte
-}
-
-func extractClusterParameters(ctx context.Context, t *testing.T, kube *Kubeclient) *ClusterParams {
-	pod, err := kube.GetHostNetworkDebugPod(ctx, t)
-	require.NoError(t, err)
-
-	execResult, err := execOnPrivilegedPod(ctx, kube, pod.Namespace, pod.Name, "cat /var/lib/kubelet/bootstrap-kubeconfig")
-	require.NoError(t, err)
-
-	bootstrapConfig := execResult.stdout.Bytes()
-
-	server, err := extractKeyValuePair("server", string(bootstrapConfig))
-	require.NoError(t, err)
-	tokens := strings.Split(server, ":")
-	if len(tokens) != 3 {
-		t.Fatalf("expected 3 tokens from fqdn %q, got %d", server, len(tokens))
-	}
-	fqdn := tokens[1][2:]
-
-	caCert, err := execOnPrivilegedPod(ctx, kube, pod.Namespace, pod.Name, "cat /etc/kubernetes/certs/ca.crt")
-	require.NoError(t, err)
-
-	cmdAPIServer, err := execOnPrivilegedPod(ctx, kube, pod.Namespace, pod.Name, "cat /etc/kubernetes/certs/apiserver.crt")
-	require.NoError(t, err)
-
-	clientKey, err := execOnPrivilegedPod(ctx, kube, pod.Namespace, pod.Name, "cat /etc/kubernetes/certs/client.key")
-	require.NoError(t, err)
-
-	return &ClusterParams{
-		CACert:         caCert.stdout.Bytes(),
-		BootstrapToken: getBootstrapToken(ctx, t, kube),
-		FQDN:           fqdn,
-		APIServerCert:  cmdAPIServer.stdout.Bytes(),
-		ClientKey:      clientKey.stdout.Bytes(),
-	}
-}
-
-func getBootstrapToken(ctx context.Context, t *testing.T, kube *Kubeclient) string {
-	secrets, err := kube.Typed.CoreV1().Secrets("kube-system").List(ctx, metav1.ListOptions{})
-	require.NoError(t, err)
-	secret := func() *corev1.Secret {
-		for _, secret := range secrets.Items {
-			if strings.HasPrefix(secret.Name, "bootstrap-token-") {
-				return &secret
-			}
-		}
-		t.Fatal("could not find secret with bootstrap-token- prefix")
-		return nil
-	}()
-	id := secret.Data["token-id"]
-	token := secret.Data["token-secret"]
-	return fmt.Sprintf("%s.%s", id, token)
-}
-
 func sshKeyName(vmPrivateIP string) string {
 	return fmt.Sprintf("sshkey%s", strings.ReplaceAll(vmPrivateIP, ".", ""))
 
@@ -133,12 +70,10 @@ func execScriptOnVm(ctx context.Context, s *Scenario, vmPrivateIP, jumpboxPodNam
 		interpreter = "powershell"
 		scriptFileName = fmt.Sprintf("script_file_%s.ps1", identifier)
 		remoteScriptFileName = fmt.Sprintf("c:/%s", scriptFileName)
-		break
 	default:
 		interpreter = "bash"
 		scriptFileName = fmt.Sprintf("script_file_%s.sh", identifier)
 		remoteScriptFileName = scriptFileName
-		break
 	}
 
 	steps := []string{
@@ -153,7 +88,7 @@ func execScriptOnVm(ctx context.Context, s *Scenario, vmPrivateIP, jumpboxPodNam
 
 	joinedSteps := strings.Join(steps, " && ")
 
-	s.T.Log(fmt.Sprintf("Executing script %[1]s using %[2]s:\n---START-SCRIPT---\n%[3]s\n---END-SCRIPT---\n", scriptFileName, interpreter, script.script))
+	s.T.Logf("Executing script %[1]s using %[2]s:\n---START-SCRIPT---\n%[3]s\n---END-SCRIPT---\n", scriptFileName, interpreter, script.script)
 
 	kube := s.Runtime.Cluster.Kube
 	execResult, err := execOnPrivilegedPod(ctx, kube, defaultNamespace, jumpboxPodName, joinedSteps)
@@ -240,9 +175,9 @@ func unprivilegedCommandArray() []string {
 func logSSHInstructions(s *Scenario) {
 	result := "SSH Instructions:"
 	if !config.Config.KeepVMSS {
-		result += fmt.Sprintf(" (VM will be automatically deleted after the test finishes, set KEEP_VMSS=true to preserve it or pause the test with a breakpoint before the test finishes)")
+		result += " (VM will be automatically deleted after the test finishes, set KEEP_VMSS=true to preserve it or pause the test with a breakpoint before the test finishes)"
 	}
-	result += fmt.Sprintf("\n========================\n")
+	result += "\n========================\n"
 	result += fmt.Sprintf("az account set --subscription %s\n", config.Config.SubscriptionID)
 	result += fmt.Sprintf("az aks get-credentials --resource-group %s --name %s --overwrite-existing\n", config.ResourceGroupName, *s.Runtime.Cluster.Model.Name)
 	result += fmt.Sprintf(`kubectl exec -it %s -- bash -c "chroot /proc/1/root /bin/bash -c '%s'"`, s.Runtime.DebugHostPod, sshString(s.Runtime.VMPrivateIP))

e2e/kube.go

@@ -2,7 +2,6 @@ package e2e
 
 import (
 	"context"
-	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"strings"
@@ -26,7 +25,6 @@ import (
 	"k8s.io/client-go/tools/clientcmd"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
-	"sigs.k8s.io/yaml"
 )
 
 type Kubeclient struct {
@@ -41,16 +39,6 @@ const (
 	podNetworkDebugAppLabel  = "debugnonhost-mariner"
 )
 
-func (k *Kubeclient) clientCertificate() string {
-	var kc map[string]any
-	if err := yaml.Unmarshal(k.KubeConfig, &kc); err != nil {
-		return ""
-	}
-	encoded := kc["users"].([]interface{})[0].(map[string]any)["user"].(map[string]any)["client-certificate-data"].(string)
-	cert, _ := base64.URLEncoding.DecodeString(encoded)
-	return string(cert)
-}
-
 func getClusterKubeClient(ctx context.Context, resourceGroupName, clusterName string) (*Kubeclient, error) {
 	data, err := getClusterKubeconfigBytes(ctx, resourceGroupName, clusterName)
 	if err != nil {

e2e/node_config.go

@@ -28,10 +28,11 @@ func getBaseNBC(t *testing.T, cluster *Cluster, vhd *config.Image) *datamodel.No
 
 	if vhd.Distro.IsWindowsDistro() {
 		nbc = baseTemplateWindows(t, config.Config.Location)
-		cert := cluster.Kube.clientCertificate()
-		nbc.ContainerService.Properties.CertificateProfile.ClientCertificate = cert
-		nbc.ContainerService.Properties.CertificateProfile.APIServerCertificate = string(cluster.ClusterParams.APIServerCert)
-		nbc.ContainerService.Properties.CertificateProfile.ClientPrivateKey = string(cluster.ClusterParams.ClientKey)
+
+		// these aren't needed since we use TLS bootstrapping instead, though windows bootstrapping expects non-empty values
+		nbc.ContainerService.Properties.CertificateProfile.ClientCertificate = "none"
+		nbc.ContainerService.Properties.CertificateProfile.ClientPrivateKey = "none"
+
 		nbc.ContainerService.Properties.ClusterID = *cluster.Model.ID
 		nbc.SubscriptionID = config.Config.SubscriptionID
 		nbc.ResourceGroupName = *cluster.Model.Properties.NodeResourceGroup
@@ -41,7 +42,6 @@ func getBaseNBC(t *testing.T, cluster *Cluster, vhd *config.Image) *datamodel.No
 	}
 
 	nbc.ContainerService.Properties.CertificateProfile.CaCertificate = string(cluster.ClusterParams.CACert)
-
 	nbc.KubeletClientTLSBootstrapToken = &cluster.ClusterParams.BootstrapToken
 	nbc.ContainerService.Properties.HostedMasterProfile.FQDN = cluster.ClusterParams.FQDN
 	nbc.ContainerService.Properties.AgentPoolProfiles[0].Distro = vhd.Distro