From f1db6fc483d22bba7184a36402503ddf4ffe9c80 Mon Sep 17 00:00:00 2001 From: Huy Mai Date: Tue, 14 Jan 2025 09:38:18 +0200 Subject: [PATCH] Only use Kustomize to install Ironic and BMO in e2e tests Kustomize has been taken into use for upgrade test, but we didn't take it into use in pivoting tests. This PR takes kustomize into use everywhere, so that we can remove the custom installIronicBMO(). Signed-off-by: Huy Mai --- test/e2e/common.go | 8 -- test/e2e/logcollector.go | 36 ++++++- test/e2e/pivoting.go | 204 +++++++++++---------------------------- 3 files changed, 87 insertions(+), 161 deletions(-) diff --git a/test/e2e/common.go b/test/e2e/common.go index fa036fe20f..3e457879b1 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -13,7 +13,6 @@ import ( "path" "path/filepath" "sort" - "strconv" "strings" "text/tabwriter" "time" @@ -83,13 +82,6 @@ func LogFromFile(logFile string) { Logf(string(data)) } -// return only the boolean value from ParseBool. -func getBool(s string) bool { - b, err := strconv.ParseBool(s) - Expect(err).ToNot(HaveOccurred()) - return b -} - // logTable print a formatted table into the e2e logs. func logTable(title string, rows [][]string) { getRowFormatted := func(row []string) string { diff --git a/test/e2e/logcollector.go b/test/e2e/logcollector.go index 2d90ba867e..dd10522e25 100644 --- a/test/e2e/logcollector.go +++ b/test/e2e/logcollector.go @@ -9,6 +9,7 @@ import ( "os/exec" "path" "path/filepath" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -116,14 +117,39 @@ func FetchClusterLogs(clusterProxy framework.ClusterProxy, outputPath string) er // Print the Pods' information to file // This does the same thing as: // kubectl --kubeconfig="${KUBECONFIG_WORKLOAD}" get pods -A - out, err := exec.Command("kubectl", "--kubeconfig", clusterProxy.GetKubeconfigPath(), "get", "pods", "-A", "-o", "wide").Output() // #nosec G204:gosec + outputFile := filepath.Join(baseDir, "pods.log") + file, err := os.Create(outputFile) if err != nil { - return fmt.Errorf("couldn't get pods: %v", err) + return fmt.Errorf("failed to create output file: %v", err) } - file := filepath.Join(baseDir, "pods.log") - err = os.WriteFile(file, out, 0600) + defer file.Close() + + // List pods across all namespaces + pods, err := clientset.CoreV1().Pods("").List(context.Background(), metav1.ListOptions{}) if err != nil { - return fmt.Errorf("couldn't write to file: %v", err) + return fmt.Errorf("failed to list pods: %v", err) + } + + // Print header to file + header := fmt.Sprintf("%-32s %-16s %-12s %-24s %-8s\n", "NAMESPACE", "NAME", "STATUS", "NODE", "AGE") + if _, err = file.WriteString(header); err != nil { + return fmt.Errorf("error writing to file: %v", err) + } + + // Iterate through pods and print information + for _, pod := range pods.Items { + age := time.Since(pod.CreationTimestamp.Time).Round(time.Second) + podInfo := fmt.Sprintf("%-32s %-16s %-12s %-24s %-8s\n", + pod.Namespace, + pod.Name, + string(pod.Status.Phase), + pod.Spec.NodeName, + age, + ) + + if _, err = file.WriteString(podInfo); err != nil { + return fmt.Errorf("error writing to file: %v", err) + } } // Get all namespaces diff --git a/test/e2e/pivoting.go b/test/e2e/pivoting.go index c792464692..fe57a638fc 100644 --- a/test/e2e/pivoting.go +++ b/test/e2e/pivoting.go @@ -7,7 +7,6 @@ import ( "os" "os/exec" "path/filepath" - "strconv" containerTypes "github.com/docker/docker/api/types/container" docker "github.com/docker/docker/client" @@ -155,42 +154,36 @@ func pivoting(ctx context.Context, inputGetter func() PivotingInput) { By("Install Ironic in the target cluster") // TODO(dtantsur): support ironic-standalone-operator - installIronicBMO(ctx, func() installIronicBMOInput { - return installIronicBMOInput{ - ManagementCluster: input.TargetCluster, - BMOPath: input.E2EConfig.GetVariable(bmoPath), - deployIronic: true, - deployBMO: false, - deployIronicTLSSetup: getBool(input.E2EConfig.GetVariable(ironicTLSSetup)), - deployIronicBasicAuth: getBool(input.E2EConfig.GetVariable(ironicBasicAuth)), - deployIronicKeepalived: getBool(input.E2EConfig.GetVariable(ironicKeepalived)), - deployIronicMariadb: getBool(input.E2EConfig.GetVariable(ironicMariadb)), - Namespace: ironicNamespaceObj.Name, - NamePrefix: input.E2EConfig.GetVariable(NamePrefix), - RestartContainerCertUpdate: getBool(input.E2EConfig.GetVariable(restartContainerCertUpdate)), - E2EConfig: input.E2EConfig, - SpecName: input.SpecName, - } + ironicDeployLogFolder := filepath.Join(os.TempDir(), "target_cluster_logs", "ironic-deploy-logs", input.TargetCluster.GetName()) + ironicKustomization := input.E2EConfig.GetVariable("IRONIC_RELEASE_LATEST") + By(fmt.Sprintf("Installing Ironic from kustomization %s on the target cluster", ironicKustomization)) + err = BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{ + Kustomization: ironicKustomization, + ClusterProxy: input.TargetCluster, + WaitForDeployment: true, + WatchDeploymentLogs: true, + LogPath: ironicDeployLogFolder, + DeploymentName: "baremetal-operator-ironic", + DeploymentNamespace: ironicNamespaceObj.Name, + WaitIntervals: input.E2EConfig.GetIntervals("default", "wait-deployment"), }) - - By("Install BMO") - installIronicBMO(ctx, func() installIronicBMOInput { - return installIronicBMOInput{ - ManagementCluster: input.TargetCluster, - BMOPath: input.E2EConfig.GetVariable(bmoPath), - deployIronic: false, - deployBMO: true, - deployIronicTLSSetup: getBool(input.E2EConfig.GetVariable(ironicTLSSetup)), - deployIronicBasicAuth: getBool(input.E2EConfig.GetVariable(ironicBasicAuth)), - deployIronicKeepalived: getBool(input.E2EConfig.GetVariable(ironicKeepalived)), - deployIronicMariadb: getBool(input.E2EConfig.GetVariable(ironicMariadb)), - Namespace: ironicNamespaceObj.Name, - NamePrefix: input.E2EConfig.GetVariable(NamePrefix), - RestartContainerCertUpdate: getBool(input.E2EConfig.GetVariable(restartContainerCertUpdate)), - E2EConfig: input.E2EConfig, - SpecName: input.SpecName, - } + Expect(err).NotTo(HaveOccurred()) + + By("Install BMO in the target cluster") + bmoDeployLogFolder := filepath.Join(os.TempDir(), "target_cluster_logs", "bmo-deploy-logs", input.TargetCluster.GetName()) + bmoKustomization := input.E2EConfig.GetVariable("BMO_RELEASE_LATEST") + By(fmt.Sprintf("Installing BMO from kustomization %s on the target cluster", bmoKustomization)) + err = BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{ + Kustomization: bmoKustomization, + ClusterProxy: input.TargetCluster, + WaitForDeployment: true, + WatchDeploymentLogs: true, + LogPath: bmoDeployLogFolder, + DeploymentName: "baremetal-operator-controller-manager", + DeploymentNamespace: ironicNamespaceObj.Name, + WaitIntervals: input.E2EConfig.GetIntervals("default", "wait-deployment"), }) + Expect(err).NotTo(HaveOccurred()) By("Add labels to BMO CRDs in the target cluster") labelBMOCRDs(ctx, input.TargetCluster) @@ -227,7 +220,6 @@ func pivoting(ctx context.Context, inputGetter func() PivotingInput) { Name: input.E2EConfig.GetVariable(NamePrefix) + "-controller-manager", } }) - pivotingCluster := framework.DiscoveryAndWaitForCluster(ctx, framework.DiscoveryAndWaitForClusterInput{ Getter: input.TargetCluster.GetClient(), Namespace: input.Namespace, @@ -268,82 +260,6 @@ func pivoting(ctx context.Context, inputGetter func() PivotingInput) { By("PIVOTING TESTS PASSED!") } -type installIronicBMOInput struct { - ManagementCluster framework.ClusterProxy - BMOPath string - deployIronic bool - deployBMO bool - deployIronicTLSSetup bool - deployIronicBasicAuth bool - deployIronicKeepalived bool - deployIronicMariadb bool - Namespace string - NamePrefix string - RestartContainerCertUpdate bool - E2EConfig *clusterctl.E2EConfig - SpecName string -} - -func installIronicBMO(ctx context.Context, inputGetter func() installIronicBMOInput) { - input := inputGetter() - - ironicHost := os.Getenv("CLUSTER_BARE_METAL_PROVISIONER_IP") - path := fmt.Sprintf("%s/tools/", input.BMOPath) - - args := []string{} - if input.deployBMO { - args = append(args, "-b") - } - if input.deployIronic { - args = append(args, "-i") - } - if input.deployIronicTLSSetup { - args = append(args, "-t") - } - if !input.deployIronicBasicAuth { - args = append(args, "-n") - } - if input.deployIronicKeepalived { - args = append(args, "-k") - } - if input.deployIronicMariadb { - args = append(args, "-m") - } - - env := []string{ - fmt.Sprintf("IRONIC_HOST=%s", ironicHost), - fmt.Sprintf("IRONIC_HOST_IP=%s", ironicHost), - fmt.Sprintf("KUBECTL_ARGS=--kubeconfig=%s", input.ManagementCluster.GetKubeconfigPath()), - fmt.Sprintf("NAMEPREFIX=%s", input.NamePrefix), - fmt.Sprintf("RESTART_CONTAINER_CERTIFICATE_UPDATED=%s", strconv.FormatBool(input.RestartContainerCertUpdate)), - "USER=ubuntu", - } - cmd := exec.Command("./deploy.sh", args...) // #nosec G204:gosec - cmd.Dir = path - cmd.Env = append(env, os.Environ()...) - - stdoutStderr, er := cmd.CombinedOutput() - Logf("%s\n", stdoutStderr) - Expect(er).ToNot(HaveOccurred(), "Failed to deploy Ironic") - deploymentNameList := []string{} - if input.deployIronic { - deploymentNameList = append(deploymentNameList, ironicSuffix) - } - if input.deployBMO { - deploymentNameList = append(deploymentNameList, "-controller-manager") - } - // Wait for the deployments to become available - clientSet := input.ManagementCluster.GetClientSet() - for _, name := range deploymentNameList { - deployment, err := clientSet.AppsV1().Deployments(input.Namespace).Get(ctx, input.NamePrefix+name, metav1.GetOptions{}) - Expect(err).ToNot(HaveOccurred(), "Unable to get the deployment %s in namespace %s \n error message: %s", input.NamePrefix+name, input.Namespace, err) - framework.WaitForDeploymentsAvailable(ctx, framework.WaitForDeploymentsAvailableInput{ - Getter: input.ManagementCluster.GetClient(), - Deployment: deployment, - }, input.E2EConfig.GetIntervals(input.SpecName, "wait-deployment")...) - } -} - type IronicDeploymentType string const ( @@ -478,27 +394,7 @@ func rePivoting(ctx context.Context, inputGetter func() RePivotingInput) { } }) - By("Reinstate BMO in Source cluster") - installIronicBMO(ctx, func() installIronicBMOInput { - return installIronicBMOInput{ - ManagementCluster: input.BootstrapClusterProxy, - BMOPath: input.E2EConfig.GetVariable(bmoPath), - deployIronic: false, - deployBMO: true, - deployIronicTLSSetup: getBool(input.E2EConfig.GetVariable(ironicTLSSetup)), - deployIronicBasicAuth: getBool(input.E2EConfig.GetVariable(ironicBasicAuth)), - deployIronicKeepalived: getBool(input.E2EConfig.GetVariable(ironicKeepalived)), - deployIronicMariadb: getBool(input.E2EConfig.GetVariable(ironicMariadb)), - Namespace: input.E2EConfig.GetVariable(ironicNamespace), - NamePrefix: input.E2EConfig.GetVariable(NamePrefix), - RestartContainerCertUpdate: getBool(input.E2EConfig.GetVariable(restartContainerCertUpdate)), - E2EConfig: input.E2EConfig, - SpecName: input.SpecName, - } - }) - By("Reinstate Ironic containers and BMH") - // TODO(mboukhalfa): add this local ironic deployment case to installIronicBMO function ephemeralCluster := os.Getenv("EPHEMERAL_CLUSTER") if ephemeralCluster == Kind { bmoPath := input.E2EConfig.GetVariable("BMOPATH") @@ -510,25 +406,37 @@ func rePivoting(ctx context.Context, inputGetter func() RePivotingInput) { Expect(err).ToNot(HaveOccurred(), "Cannot run local ironic") } else { By("Install Ironic in the bootstrap cluster") - installIronicBMO(ctx, func() installIronicBMOInput { - return installIronicBMOInput{ - ManagementCluster: input.BootstrapClusterProxy, - BMOPath: input.E2EConfig.GetVariable(bmoPath), - deployIronic: true, - deployBMO: false, - deployIronicTLSSetup: getBool(input.E2EConfig.GetVariable(ironicTLSSetup)), - deployIronicBasicAuth: getBool(input.E2EConfig.GetVariable(ironicBasicAuth)), - deployIronicKeepalived: getBool(input.E2EConfig.GetVariable(ironicKeepalived)), - deployIronicMariadb: getBool(input.E2EConfig.GetVariable(ironicMariadb)), - Namespace: input.E2EConfig.GetVariable(ironicNamespace), - NamePrefix: input.E2EConfig.GetVariable(NamePrefix), - RestartContainerCertUpdate: getBool(input.E2EConfig.GetVariable(restartContainerCertUpdate)), - E2EConfig: input.E2EConfig, - SpecName: input.SpecName, - } + ironicKustomization := input.E2EConfig.GetVariable("IRONIC_RELEASE_LATEST") + ironicDeployLogFolder := filepath.Join(os.TempDir(), "source_cluster_logs", "ironic-deploy-logs", input.TargetCluster.GetName()) + err = BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{ + Kustomization: ironicKustomization, + ClusterProxy: input.BootstrapClusterProxy, + WaitForDeployment: true, + WatchDeploymentLogs: true, + LogPath: ironicDeployLogFolder, + DeploymentName: "baremetal-operator-ironic", + DeploymentNamespace: input.E2EConfig.GetVariable(ironicNamespace), + WaitIntervals: input.E2EConfig.GetIntervals("default", "wait-deployment"), }) + Expect(err).NotTo(HaveOccurred()) } + By("Reinstate BMO in Source cluster") + bmoKustomization := input.E2EConfig.GetVariable("BMO_RELEASE_LATEST") + bmoDeployLogFolder := filepath.Join(os.TempDir(), "source_cluster_logs", "bmo-deploy-logs", input.TargetCluster.GetName()) + By(fmt.Sprintf("Installing BMO from kustomization %s on the source cluster", bmoKustomization)) + err = BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{ + Kustomization: bmoKustomization, + ClusterProxy: input.BootstrapClusterProxy, + WaitForDeployment: true, + WatchDeploymentLogs: true, + LogPath: bmoDeployLogFolder, + DeploymentName: "baremetal-operator-controller-manager", + DeploymentNamespace: input.E2EConfig.GetVariable(ironicNamespace), + WaitIntervals: input.E2EConfig.GetIntervals("default", "wait-deployment"), + }) + Expect(err).NotTo(HaveOccurred()) + By("Ensure API servers are stable before doing move") // Nb. This check was introduced to prevent doing move to self-hosted in an aggressive way and thus avoid flakes. // More specifically, it was observed that the test was failing to get objects from the API server during move, so now