From 70cbb85beef269131e15612b982c1d7acb678a12 Mon Sep 17 00:00:00 2001 From: Mohammed Boukhalfa Date: Thu, 31 Oct 2024 15:30:59 +0200 Subject: [PATCH] Add scalability test by reusing capi scale e2e test with clusterclass Signed-off-by: Mohammed Boukhalfa --- Makefile | 2 + scripts/ci-e2e.sh | 12 +- scripts/environment.sh | 7 +- test/e2e/config/e2e_conf.yaml | 8 +- test/e2e/data/fkas/kustomization.yaml | 2 + test/e2e/data/fkas/resources.yaml | 77 ++++++ .../cluster-with-topology.yaml | 107 ++++++++ .../cluster-with-topology/kustomization.yaml | 2 + .../kustomization.yaml | 2 + .../clusterclass-metal3.yaml | 210 +++++++++++++++ .../clusterclass-metal3/kustomization.yaml | 2 + test/e2e/scalability_test.go | 243 ++++++++++++++++-- test/go.mod | 6 +- test/go.sum | 12 +- 14 files changed, 652 insertions(+), 40 deletions(-) create mode 100644 test/e2e/data/fkas/kustomization.yaml create mode 100644 test/e2e/data/fkas/resources.yaml create mode 100644 test/e2e/data/infrastructure-metal3/bases/cluster-with-topology/cluster-with-topology.yaml create mode 100644 test/e2e/data/infrastructure-metal3/bases/cluster-with-topology/kustomization.yaml create mode 100644 test/e2e/data/infrastructure-metal3/cluster-template-centos-fake/kustomization.yaml create mode 100644 test/e2e/data/infrastructure-metal3/clusterclass-metal3/clusterclass-metal3.yaml create mode 100644 test/e2e/data/infrastructure-metal3/clusterclass-metal3/kustomization.yaml diff --git a/Makefile b/Makefile index b80e9172c0..850b3f4624 100644 --- a/Makefile +++ b/Makefile @@ -173,6 +173,8 @@ E2E_TEMPLATES_DIR ?= $(ROOT_DIR)/test/e2e/data/infrastructure-metal3 cluster-templates: $(KUSTOMIZE) ## Generate cluster templates $(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/cluster-template-ubuntu > $(E2E_OUT_DIR)/cluster-template-ubuntu.yaml $(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/cluster-template-centos > $(E2E_OUT_DIR)/cluster-template-centos.yaml + $(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/cluster-template-centos-fake > $(E2E_OUT_DIR)/cluster-template-centos-fake.yaml + $(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/clusterclass-metal3 > $(E2E_OUT_DIR)/clusterclass-metal3.yaml $(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/cluster-template-upgrade-workload > $(E2E_OUT_DIR)/cluster-template-upgrade-workload.yaml $(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/cluster-template-centos-md-remediation > $(E2E_OUT_DIR)/cluster-template-centos-md-remediation.yaml $(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/cluster-template-ubuntu-md-remediation > $(E2E_OUT_DIR)/cluster-template-ubuntu-md-remediation.yaml diff --git a/scripts/ci-e2e.sh b/scripts/ci-e2e.sh index fcb461027a..168eff07e3 100755 --- a/scripts/ci-e2e.sh +++ b/scripts/ci-e2e.sh @@ -34,7 +34,7 @@ export KUBERNETES_VERSION=${KUBERNETES_VERSION} export IMAGE_OS=${IMAGE_OS} export FORCE_REPO_UPDATE="false" EOF -# if running a clusterctl-upgrade test skip apply bmhs in dev-env +# if running a scalability test skip apply bmhs in dev-env and run fakeIPA if [[ ${GINKGO_FOCUS:-} == "clusterctl-upgrade" ]]; then echo 'export SKIP_APPLY_BMH="true"' >>"${M3_DEV_ENV_PATH}/config_${USER}.sh" fi @@ -42,10 +42,16 @@ if [[ ${GINKGO_FOCUS:-} == "features" ]]; then mkdir -p "$CAPI_CONFIG_FOLDER" echo "ENABLE_BMH_NAME_BASED_PREALLOCATION: true" >"$CAPI_CONFIG_FOLDER/clusterctl.yaml" fi -# if running a scalability test skip apply bmhs in dev-env and run fakeIPA +# if running a scalability tests, configure dev-env with fakeIPA if [[ ${GINKGO_FOCUS:-} == "scalability" ]]; then - echo 'export SKIP_APPLY_BMH="true"' >>"${M3_DEV_ENV_PATH}/config_${USER}.sh" echo 'export NODES_PLATFORM="fake"' >>"${M3_DEV_ENV_PATH}/config_${USER}.sh" + echo 'export SKIP_APPLY_BMH="true"' >>"${M3_DEV_ENV_PATH}/config_${USER}.sh" + mkdir -p "$CAPI_CONFIG_FOLDER" + echo 'CLUSTER_TOPOLOGY: true' >"$CAPI_CONFIG_FOLDER/clusterctl.yaml" + echo 'export EPHEMERAL_CLUSTER="minikube"' >>"${M3_DEV_ENV_PATH}/config_${USER}.sh" +else + # Don't run scalability tests if not asked for. + export GINKGO_SKIP="${GINKGO_SKIP:-} scalability" fi # Run make devenv to boot the source cluster pushd "${M3_DEV_ENV_PATH}" || exit 1 diff --git a/scripts/environment.sh b/scripts/environment.sh index 02c47bd780..7585136cdd 100644 --- a/scripts/environment.sh +++ b/scripts/environment.sh @@ -64,8 +64,11 @@ fi # Scalability test environment vars and config if [[ ${GINKGO_FOCUS:-} == "scalability" ]]; then - export NUM_NODES=${NUM_NODES:-"100"} - export BMH_BATCH_SIZE=${BMH_BATCH_SIZE:-"20"} + export NUM_NODES=${NUM_NODES:-"10"} + export BMH_BATCH_SIZE=${BMH_BATCH_SIZE:-"2"} + export CONTROL_PLANE_MACHINE_COUNT=${CONTROL_PLANE_MACHINE_COUNT:-"1"} + export WORKER_MACHINE_COUNT=${WORKER_MACHINE_COUNT:-"0"} + export KUBERNETES_VERSION_UPGRADE_FROM=${FROM_K8S_VERSION} fi # Integration test environment vars and config diff --git a/test/e2e/config/e2e_conf.yaml b/test/e2e/config/e2e_conf.yaml index aff17f9b9f..beab3cf66e 100644 --- a/test/e2e/config/e2e_conf.yaml +++ b/test/e2e/config/e2e_conf.yaml @@ -188,8 +188,13 @@ providers: targetName: "cluster-template-ubuntu.yaml" - sourcePath: "../_out/cluster-template-centos.yaml" targetName: "cluster-template-centos.yaml" + - sourcePath: "../_out/cluster-template-centos-fake.yaml" + targetName: "cluster-template-centos-fake.yaml" + - sourcePath: "../_out/cluster-template-centos-fake.yaml" + targetName: "cluster-template-ubuntu-fake.yaml" - sourcePath: "../_out/clusterclass.yaml" targetName: "clusterclass-test-clusterclass.yaml" + - sourcePath: "../_out/clusterclass-metal3.yaml" - sourcePath: "../_out/cluster-template-centos-md-remediation.yaml" targetName: "cluster-template-centos-md-remediation.yaml" - sourcePath: "../_out/cluster-template-ubuntu-md-remediation.yaml" @@ -246,7 +251,8 @@ variables: BMO_RELEASE_0.6: "data/bmo-deployment/overlays/release-0.6" BMO_RELEASE_0.8: "data/bmo-deployment/overlays/release-0.8" BMO_RELEASE_LATEST: "data/bmo-deployment/overlays/release-latest" - + FKAS_RELEASE_LATEST: "data/fkas" + intervals: default/wait-controllers: ["10m", "10s"] default/wait-cluster: ["20m", "30s"] # The second time to check the availibility of the cluster should happen late, so kcp object has time to be created diff --git a/test/e2e/data/fkas/kustomization.yaml b/test/e2e/data/fkas/kustomization.yaml new file mode 100644 index 0000000000..b6b8eeb903 --- /dev/null +++ b/test/e2e/data/fkas/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- resources.yaml diff --git a/test/e2e/data/fkas/resources.yaml b/test/e2e/data/fkas/resources.yaml new file mode 100644 index 0000000000..920ae9bb00 --- /dev/null +++ b/test/e2e/data/fkas/resources.yaml @@ -0,0 +1,77 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: fkas-system +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: metal3-fkas-sa + namespace: fkas-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metal3-fkas-role +rules: +- apiGroups: ["metal3.io", "infrastructure.cluster.x-k8s.io"] + resources: ["baremetalhosts", "metal3machines"] + verbs: ["get", "list", "watch"] +- apiGroups: ["cluster.x-k8s.io"] + resources: ["machines"] + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metal3-fkas-rolebinding +subjects: +- kind: ServiceAccount + name: metal3-fkas-sa + namespace: fkas-system +roleRef: + kind: ClusterRole + name: metal3-fkas-role + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: metal3-fkas-system + namespace: fkas-system +spec: + replicas: 1 + selector: + matchLabels: + app: metal3-fkas-system + template: + metadata: + labels: + app: metal3-fkas-system + spec: + serviceAccountName: metal3-fkas-sa + hostNetwork: true + containers: + - name: metal3-fkas-reconciler + image: quay.io/metal3-io/metal3-fkas:latest + imagePullPolicy: IfNotPresent + command: ["/reconciler"] + env: + - name: DEBUG + value: "true" + - image: quay.io/metal3-io/metal3-fkas:latest + imagePullPolicy: IfNotPresent + ports: + - containerPort: 3333 + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: DEBUG + value: "true" + name: metal3-fkas diff --git a/test/e2e/data/infrastructure-metal3/bases/cluster-with-topology/cluster-with-topology.yaml b/test/e2e/data/infrastructure-metal3/bases/cluster-with-topology/cluster-with-topology.yaml new file mode 100644 index 0000000000..74d3a5066a --- /dev/null +++ b/test/e2e/data/infrastructure-metal3/bases/cluster-with-topology/cluster-with-topology.yaml @@ -0,0 +1,107 @@ +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cni: ${CLUSTER_NAME}-crs-0 + name: ${CLUSTER_NAME} + namespace: ${NAMESPACE} +spec: + clusterNetwork: + pods: + cidrBlocks: ["${POD_CIDR}"] + services: + cidrBlocks: ["${SERVICE_CIDR}"] + topology: + class: metal3 + version: ${KUBERNETES_VERSION} + controlPlane: + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + workers: + machineDeployments: + - class: worker + name: ${CLUSTER_NAME}-machine + replicas: ${WORKER_MACHINE_COUNT} + variables: + - name: image + value: + checksum: ${IMAGE_RAW_CHECKSUM} + checksumType: ${IMAGE_CHECKSUM_TYPE} + format: raw + url: ${IMAGE_RAW_URL} + - name: controlPlaneEndpoint + value: + host: CLUSTER_APIENDPOINT_HOST_HOLDER + port: CLUSTER_APIENDPOINT_PORT_HOLDER + - name: controlPlaneDataTemplate + value: ${CLUSTER_NAME}-controlplane-template +--- +apiVersion: ipam.metal3.io/v1alpha1 +kind: IPPool +metadata: + name: ${CLUSTER_NAME}-baremetalv4-pool +spec: + clusterName: ${CLUSTER_NAME} + gateway: 192.168.111.1 + namePrefix: ${CLUSTER_NAME}-bmv4 + pools: + - end: 192.168.111.240 + start: 192.168.111.201 + prefix: 24 +--- +apiVersion: ipam.metal3.io/v1alpha1 +kind: IPPool +metadata: + name: ${CLUSTER_NAME}-provisioning-pool +spec: + clusterName: ${CLUSTER_NAME} + namePrefix: ${CLUSTER_NAME}-prov + pools: + - end: 172.22.0.240 + start: 172.22.0.201 + prefix: 24 +--- +apiVersion: infrastructure.cluster.x-k8s.io/${CAPM3_VERSION} +kind: Metal3DataTemplate +metadata: + name: ${CLUSTER_NAME}-controlplane-template +spec: + clusterName: ${CLUSTER_NAME} + metaData: + ipAddressesFromIPPool: + - key: provisioningIP + name: ${CLUSTER_NAME}-provisioning-pool + objectNames: + - key: name + object: machine + - key: local-hostname + object: machine + - key: local_hostname + object: machine + prefixesFromIPPool: + - key: provisioningCIDR + name: ${CLUSTER_NAME}-provisioning-pool + networkData: + links: + ethernets: + - id: enp1s0 + macAddress: + fromHostInterface: enp1s0 + type: phy + - id: enp2s0 + macAddress: + fromHostInterface: enp2s0 + type: phy + networks: + ipv4: + - id: baremetalv4 + ipAddressFromIPPool: ${CLUSTER_NAME}-baremetalv4-pool + link: enp2s0 + routes: + - gateway: + fromIPPool: ${CLUSTER_NAME}-baremetalv4-pool + network: 0.0.0.0 + prefix: 0 + services: + dns: + - 8.8.8.8 diff --git a/test/e2e/data/infrastructure-metal3/bases/cluster-with-topology/kustomization.yaml b/test/e2e/data/infrastructure-metal3/bases/cluster-with-topology/kustomization.yaml new file mode 100644 index 0000000000..ea38dfdd13 --- /dev/null +++ b/test/e2e/data/infrastructure-metal3/bases/cluster-with-topology/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- cluster-with-topology.yaml diff --git a/test/e2e/data/infrastructure-metal3/cluster-template-centos-fake/kustomization.yaml b/test/e2e/data/infrastructure-metal3/cluster-template-centos-fake/kustomization.yaml new file mode 100644 index 0000000000..c7031d7b2d --- /dev/null +++ b/test/e2e/data/infrastructure-metal3/cluster-template-centos-fake/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- ../bases/cluster-with-topology diff --git a/test/e2e/data/infrastructure-metal3/clusterclass-metal3/clusterclass-metal3.yaml b/test/e2e/data/infrastructure-metal3/clusterclass-metal3/clusterclass-metal3.yaml new file mode 100644 index 0000000000..0bb4bf238c --- /dev/null +++ b/test/e2e/data/infrastructure-metal3/clusterclass-metal3/clusterclass-metal3.yaml @@ -0,0 +1,210 @@ +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: ClusterClass +metadata: + name: metal3 +spec: + variables: + - name: controlPlaneEndpoint + required: true + schema: + openAPIV3Schema: + type: object + properties: + host: + type: string + port: + type: integer + - name: image + schema: + openAPIV3Schema: + type: object + properties: + checksum: + type: string + checksumType: + type: string + format: + type: string + url: + type: string + - name: controlPlaneDataTemplate + schema: + openAPIV3Schema: + type: string + patches: + - name: controlPlaneEndpointSub + description: Overrides controlPlaneEndpoint data of Metal3ClusterTemplate used by the cluster + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: Metal3ClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: replace + path: /spec/template/spec/controlPlaneEndpoint + valueFrom: + variable: controlPlaneEndpoint + - name: controlPlaneImageSub + description: Overrides image data for worker nodes of control plane node + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: Metal3MachineTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: replace + path: /spec/template/spec/image/checksum + valueFrom: + variable: image.checksum + - op: replace + path: /spec/template/spec/image/checksumType + valueFrom: + variable: image.checksumType + - op: replace + path: /spec/template/spec/image/format + valueFrom: + variable: image.format + - op: replace + path: /spec/template/spec/image/url + valueFrom: + variable: image.url + - name: controlPlaneDataTemplateSub + description: Overrides data-template for control plane nodes + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: Metal3MachineTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: replace + path: /spec/template/spec/dataTemplate/name + valueFrom: + variable: controlPlaneDataTemplate + controlPlane: + ref: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlaneTemplate + name: metal3-control-plane + machineInfrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: Metal3MachineTemplate + name: metal3-control-plane + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: Metal3ClusterTemplate + name: metal3-cluster + workers: + machineDeployments: + - class: worker + template: + metadata: + labels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + nodepool: nodepool-0 + bootstrap: + ref: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: metal3-default-worker-bootstraptemplate + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: Metal3MachineTemplate + name: metal3-default-worker-machinetemplate +--- +apiVersion: infrastructure.cluster.x-k8s.io/${CAPM3_VERSION} +kind: Metal3ClusterTemplate +metadata: + name: metal3-cluster +spec: + template: + spec: + controlPlaneEndpoint: + host: ${CLUSTER_APIENDPOINT_HOST} + port: ${CLUSTER_APIENDPOINT_PORT} + noCloudProvider: true +--- +apiVersion: controlplane.cluster.x-k8s.io/${CAPI_VERSION} +kind: KubeadmControlPlaneTemplate +metadata: + name: metal3-control-plane +spec: + template: + spec: + kubeadmConfigSpec: + clusterConfiguration: {} + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + cgroup-driver: systemd + container-runtime-endpoint: unix:///var/run/crio/crio.sock + feature-gates: AllAlpha=false + node-labels: metal3.io/uuid={{ ds.meta_data.uuid }} + provider-id: ${PROVIDER_ID_FORMAT} + runtime-request-timeout: 5m + name: '{{ ds.meta_data.name }}' + joinConfiguration: + controlPlane: {} + nodeRegistration: + kubeletExtraArgs: + cgroup-driver: systemd + container-runtime-endpoint: unix:///var/run/crio/crio.sock + feature-gates: AllAlpha=false + node-labels: metal3.io/uuid={{ ds.meta_data.uuid }} + provider-id: ${PROVIDER_ID_FORMAT} + runtime-request-timeout: 5m + name: '{{ ds.meta_data.name }}' +--- +apiVersion: infrastructure.cluster.x-k8s.io/${CAPM3_VERSION} +kind: Metal3MachineTemplate +metadata: + name: metal3-control-plane +spec: + template: + spec: + dataTemplate: + name: metal3-controlplane-template + image: + checksum: ${IMAGE_RAW_CHECKSUM} + checksumType: ${IMAGE_CHECKSUM_TYPE} + format: raw + url: ${IMAGE_RAW_URL} +--- +apiVersion: bootstrap.cluster.x-k8s.io/${CAPI_VERSION} +kind: KubeadmConfigTemplate +metadata: + name: metal3-default-worker-bootstraptemplate +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cgroup-driver: systemd + container-runtime-endpoint: unix:///var/run/crio/crio.sock + feature-gates: AllAlpha=false + node-labels: metal3.io/uuid={{ ds.meta_data.uuid }} + provider-id: ${PROVIDER_ID_FORMAT} + runtime-request-timeout: 5m + name: '{{ ds.meta_data.name }}' +--- +apiVersion: infrastructure.cluster.x-k8s.io/${CAPM3_VERSION} +kind: Metal3MachineTemplate +metadata: + name: metal3-default-worker-machinetemplate +spec: + template: + spec: + dataTemplate: + name: metal3-controlplane-template + image: + checksum: ${IMAGE_RAW_CHECKSUM} + checksumType: ${IMAGE_CHECKSUM_TYPE} + format: raw + url: ${IMAGE_RAW_URL} diff --git a/test/e2e/data/infrastructure-metal3/clusterclass-metal3/kustomization.yaml b/test/e2e/data/infrastructure-metal3/clusterclass-metal3/kustomization.yaml new file mode 100644 index 0000000000..72b6033eee --- /dev/null +++ b/test/e2e/data/infrastructure-metal3/clusterclass-metal3/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- clusterclass-metal3.yaml diff --git a/test/e2e/scalability_test.go b/test/e2e/scalability_test.go index 6c75a030f1..c7869c846d 100644 --- a/test/e2e/scalability_test.go +++ b/test/e2e/scalability_test.go @@ -1,57 +1,248 @@ package e2e import ( + "bytes" + "context" + "encoding/json" "fmt" + "io" + "io/ioutil" + "net/http" "os" "path/filepath" + "regexp" "strconv" + "strings" bmov1alpha1 "github.com/metal3-io/baremetal-operator/apis/metal3.io/v1alpha1" + infrav1 "github.com/metal3-io/cluster-api-provider-metal3/api/v1beta1" + ipamv1 "github.com/metal3-io/ip-address-manager/api/v1alpha1" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/utils/ptr" + capi_e2e "sigs.k8s.io/cluster-api/test/e2e" + "sigs.k8s.io/cluster-api/test/framework" + "sigs.k8s.io/cluster-api/util/yaml" "sigs.k8s.io/controller-runtime/pkg/client" ) /* - * Apply scaled number of BMHs per batches and wait for the them to become available - * The test pass when all the BMHs become available + * This test apply cluster templates on fake environments to test scalability + * apply scaled number of BMHs per batches and wait for the them to become available + * When all the BMHs become available it apply the cluster templates */ -var _ = Describe("When testing scalability with fakeIPA [scalability]", Label("scalability"), func() { - scalabilityTest() +type ClusterRequest struct { + ClusterName string `json:"cluster"` + Namespace string `json:"namespace"` +} + +type ClusterAPIServer struct { + Host string + Port int +} + +var _ = Describe("When testing scalability with fakeIPA and FKAS [scalability]", Label("scalability"), func() { + BeforeEach(func() { + osType := strings.ToLower(os.Getenv("OS")) + Expect(osType).ToNot(Equal("")) + validateGlobals(specName) + specName = "scale" + namespace = "scale" + numberOfWorkers = int(*e2eConfig.GetInt32PtrVariable("WORKER_MACHINE_COUNT")) + numberOfControlplane = int(*e2eConfig.GetInt32PtrVariable("CONTROL_PLANE_MACHINE_COUNT")) + // We need to override clusterctl apply log folder to avoid getting our credentials exposed. + clusterctlLogFolder = filepath.Join(os.TempDir(), "clusters", bootstrapClusterProxy.GetName()) + createFKASResources() + imageURL, imageChecksum := EnsureImage("v1.30.0") + os.Setenv("IMAGE_RAW_CHECKSUM", imageChecksum) + os.Setenv("IMAGE_RAW_URL", imageURL) + }) + + // Create the cluster + capi_e2e.ScaleSpec(ctx, func() capi_e2e.ScaleSpecInput { + return capi_e2e.ScaleSpecInput{ + E2EConfig: e2eConfig, + ClusterctlConfigPath: clusterctlConfigPath, + InfrastructureProvider: ptr.To("metal3"), + BootstrapClusterProxy: bootstrapClusterProxy, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + SkipUpgrade: true, + ClusterCount: ptr.To[int64](5), + Concurrency: ptr.To[int64](5), + Flavor: ptr.To(fmt.Sprintf("%s-fake", osType)), + ControlPlaneMachineCount: ptr.To[int64](int64(numberOfControlplane)), + MachineDeploymentCount: ptr.To[int64](0), + WorkerMachineCount: ptr.To[int64](int64(numberOfWorkers)), + PostScaleClusterNamespaceCreated: postScaleClusterNamespaceCreated, + DeployClusterInSeparateNamespaces: true, + } + }) + + AfterEach(func() { + FKASKustomization := e2eConfig.GetVariable("FKAS_RELEASE_LATEST") + By(fmt.Sprintf("Removing FKAS from kustomization %s from the bootsrap cluster", FKASKustomization)) + err := BuildAndRemoveKustomization(ctx, FKASKustomization, bootstrapClusterProxy) + Expect(err).NotTo(HaveOccurred()) + }) }) -func scalabilityTest() { - It("Should apply BMHs per batches and wait for them to become available", func() { - numNodes, _ := strconv.Atoi(e2eConfig.GetVariable("NUM_NODES")) - batch, _ := strconv.Atoi(e2eConfig.GetVariable("BMH_BATCH_SIZE")) - Logf("Starting scalability test") - bootstrapClient := bootstrapClusterProxy.GetClient() - ListBareMetalHosts(ctx, bootstrapClient, client.InNamespace(namespace)) +func registerFKASCluster(ctx context.Context, cn string, ns string) *ClusterAPIServer { + /* + * this function reads the certificate from the cluster secrets + */ + + fkasCluster := ClusterRequest{ + ClusterName: cn, + Namespace: ns, + } + jsonData, err := json.Marshal(fkasCluster) + Expect(err).NotTo(HaveOccurred()) + registerEndpoint := "http://172.22.0.2:3333/register" + // Create the HTTP POST request with the context + req, err := http.NewRequestWithContext(ctx, http.MethodPost, registerEndpoint, http.NoBody) + Expect(err).NotTo(HaveOccurred()) + + req.Header.Set("Content-Type", "application/json") + req.Body = io.NopCloser(bytes.NewReader(jsonData)) + + // Use the default HTTP client to send the request + client := &http.Client{} + resp, err := client.Do(req) + Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + Expect(err).NotTo(HaveOccurred()) + var endpoint ClusterAPIServer + err = json.Unmarshal(body, &endpoint) + Expect(err).NotTo(HaveOccurred()) + return &endpoint +} + +func createFKASResources() { + FKASDeployLogFolder := filepath.Join(os.TempDir(), "fkas-deploy-logs", bootstrapClusterProxy.GetName()) + FKASKustomization := e2eConfig.GetVariable("FKAS_RELEASE_LATEST") + By(fmt.Sprintf("Installing FKAS with kustomization %s to the bootstrap cluster", FKASKustomization)) + err := BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{ + Kustomization: FKASKustomization, + ClusterProxy: bootstrapClusterProxy, + WaitForDeployment: true, + WatchDeploymentLogs: true, + LogPath: FKASDeployLogFolder, + DeploymentName: "metal3-fkas-system", + DeploymentNamespace: "fkas-system", + WaitIntervals: e2eConfig.GetIntervals("default", "wait-deployment"), + }) + Expect(err).NotTo(HaveOccurred()) +} + +func LogToFile(logFile string, data []byte) { + err := ioutil.WriteFile(filepath.Clean(logFile), data, 0600) + Expect(err).ToNot(HaveOccurred(), "Cannot log to file") +} + +// the function to create the bmh needed in the namespace and call fkas to prepare a fakecluster and update templates accordingly. +func postScaleClusterNamespaceCreated(clusterProxy framework.ClusterProxy, clusterNamespace string, clusterName string, baseClusterClassYAML []byte, baseClusterTemplateYAML []byte) (clusterClassTemlate []byte, clusterTemplate []byte) { + c := clusterProxy.GetClient() + + getClusterID := func(clusterName string) (index int) { + re := regexp.MustCompile("[0-9]+$") + index, _ = strconv.Atoi(string((re.Find([]byte(clusterName))))) + return + } + + getBmhsCountNeeded := func() (sum int) { + numberOfWorkers = int(*e2eConfig.GetInt32PtrVariable("WORKER_MACHINE_COUNT")) + numberOfControlplane = int(*e2eConfig.GetInt32PtrVariable("CONTROL_PLANE_MACHINE_COUNT")) + sum = numberOfWorkers + numberOfControlplane + return + } + + getBmhsFromToIndex := func(clusterIndex int, bmhCount int) (from int, to int) { + to = clusterIndex*bmhCount - 1 + from = to - bmhCount + 1 + return + } + + applyBmhsByBatch := func(batchSize int, from int, to int) { applyBatchBmh := func(from int, to int) { + bmhsNameList := make([]string, 0, to-from+1) Logf("Apply BMH batch from node_%d to node_%d", from, to) for i := from; i < to+1; i++ { + bmhsNameList = append(bmhsNameList, fmt.Sprintf("node-%d", i)) resource, err := os.ReadFile(filepath.Join(workDir, fmt.Sprintf("bmhs/node_%d.yaml", i))) Expect(err).ShouldNot(HaveOccurred()) - Expect(CreateOrUpdateWithNamespace(ctx, bootstrapClusterProxy, resource, namespace)).ShouldNot(HaveOccurred()) + Expect(CreateOrUpdateWithNamespace(ctx, clusterProxy, resource, clusterNamespace)).ShouldNot(HaveOccurred()) } - Logf("Wait for batch from node_%d to node_%d to become available", from, to) - WaitForNumBmhInState(ctx, bmov1alpha1.StateAvailable, WaitForNumInput{ - Client: bootstrapClient, - Options: []client.ListOption{client.InNamespace(namespace)}, - Replicas: to + 1, - Intervals: e2eConfig.GetIntervals(specName, "wait-bmh-available"), - }) + // I need to get a list of bmh names we are getting bmhlist to avoid http request one by one + // TODO (mboukhalfa) if the clusters are using the same namespace then might pickup the bmh from the list + // that we are waiting to become available aby another cluster then the bmh number avialble will never reached + Logf("Waiting for BMHs from node_%d to node_%d to become available", from, to) + Eventually(func(g Gomega) { + bmhList := bmov1alpha1.BareMetalHostList{} + g.Expect(c.List(ctx, &bmhList, []client.ListOption{client.InNamespace(clusterNamespace)}...)).To(Succeed()) + g.Expect(FilterAvialableBmhsName(bmhList.Items, bmhsNameList, bmov1alpha1.StateAvailable)).To(HaveLen(to - from + 1)) + }, e2eConfig.GetIntervals(specName, "wait-bmh-available")...).Should(Succeed()) + ListBareMetalHosts(ctx, c, []client.ListOption{client.InNamespace(clusterNamespace)}...) } - - for i := 0; i < numNodes; i += batch { - if i+batch > numNodes { - applyBatchBmh(i, numNodes-1) + for i := from; i <= to; i += batchSize { + if i+batchSize > to { + applyBatchBmh(i, to) break } - applyBatchBmh(i, i+batch-1) + applyBatchBmh(i, i+batchSize-1) } + } + index := getClusterID(clusterName) + cn := getBmhsCountNeeded() + f, t := getBmhsFromToIndex(index, cn) + batch, _ := strconv.Atoi(e2eConfig.GetVariable("BMH_BATCH_SIZE")) + applyBmhsByBatch(batch, f, t) - By("SCALABILITY TEST PASSED!") - }) + newClusterEndpoint := registerFKASCluster(ctx, clusterName, clusterNamespace) + clusterTemplateYAML := bytes.ReplaceAll(baseClusterTemplateYAML, []byte("CLUSTER_APIENDPOINT_HOST_HOLDER"), []byte(newClusterEndpoint.Host)) + clusterTemplateYAML = bytes.ReplaceAll(clusterTemplateYAML, []byte("CLUSTER_APIENDPOINT_PORT_HOLDER"), []byte(strconv.Itoa(newClusterEndpoint.Port))) + + clusterClassYAML, clusterTemplateYAML := extractDataTemplateIppool(baseClusterClassYAML, clusterTemplateYAML) + Logf("save " + clusterName + " cluster in a file") + LogToFile("/tmp/"+clusterName+"-cluster.log", clusterTemplateYAML) + Logf("save " + clusterName + " clusterclass in a file") + LogToFile("/tmp/"+clusterName+"-clusterclass.log", clusterClassYAML) + return clusterClassYAML, clusterTemplateYAML +} + +func extractDataTemplateIppool(baseClusterClassYAML []byte, baseClusterTemplateYAML []byte) (clusterClassTemlate []byte, clusterTemplate []byte) { + clusterClassObjs, err := yaml.ToUnstructured(baseClusterClassYAML) + Expect(err).ToNot(HaveOccurred()) + newClusterClassObjs := []unstructured.Unstructured{} + clusterObjs, err := yaml.ToUnstructured(baseClusterTemplateYAML) + Expect(err).ToNot(HaveOccurred()) + for _, obj := range clusterClassObjs { + if obj.GroupVersionKind().GroupKind() == ipamv1.GroupVersion.WithKind("IPPool").GroupKind() || obj.GroupVersionKind().GroupKind() == infrav1.GroupVersion.WithKind("Metal3DataTemplate").GroupKind() { + Logf("move %v cluster", obj.GroupVersionKind().GroupKind()) + clusterObjs = append(clusterObjs, obj) + } else { + newClusterClassObjs = append(newClusterClassObjs, obj) + } + } + clusterYAML, err := yaml.FromUnstructured(clusterObjs) + Expect(err).ToNot(HaveOccurred()) + clusterClassYAML, err := yaml.FromUnstructured(newClusterClassObjs) + Expect(err).ToNot(HaveOccurred()) + return clusterClassYAML, clusterYAML +} + +// FilterAvialableBmhsName returns a filtered list of BaremetalHost objects in certain provisioning state. +func FilterAvialableBmhsName(bmhs []bmov1alpha1.BareMetalHost, bmhsNameList []string, state bmov1alpha1.ProvisioningState) (result []bmov1alpha1.BareMetalHost) { + for _, bmh := range bmhs { + for _, name := range bmhsNameList { + if bmh.Name == name && bmh.Status.Provisioning.State == state { + result = append(result, bmh) + } + } + } + return } diff --git a/test/go.mod b/test/go.mod index f46fc3d6ad..aaa7489650 100644 --- a/test/go.mod +++ b/test/go.mod @@ -4,7 +4,7 @@ go 1.22.8 require ( github.com/blang/semver v3.5.1+incompatible - github.com/docker/docker v27.3.1+incompatible + github.com/docker/docker v27.4.0+incompatible github.com/jinzhu/copier v0.4.0 github.com/metal3-io/baremetal-operator/apis v0.8.0 github.com/metal3-io/cluster-api-provider-metal3/api v0.0.0 @@ -31,6 +31,8 @@ require ( replace github.com/metal3-io/cluster-api-provider-metal3/api => ./../api +replace sigs.k8s.io/cluster-api/test => github.com/Nordix/cluster-api/test v1.0.1-0.20241211080404-d3cdd1ac1635 + require ( dario.cat/mergo v1.0.1 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect @@ -139,7 +141,7 @@ require ( go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect - golang.org/x/net v0.31.0 // indirect + golang.org/x/net v0.32.0 // indirect golang.org/x/oauth2 v0.24.0 // indirect golang.org/x/sync v0.10.0 // indirect golang.org/x/sys v0.28.0 // indirect diff --git a/test/go.sum b/test/go.sum index ab756c5221..e1b6a43ff0 100644 --- a/test/go.sum +++ b/test/go.sum @@ -14,6 +14,8 @@ github.com/Masterminds/sprig/v3 v3.3.0 h1:mQh0Yrg1XPo6vjYXgtf5OtijNAKJRNcTdOOGZe github.com/Masterminds/sprig/v3 v3.3.0/go.mod h1:Zy1iXRYNqNLUolqCpL4uhk6SHUMAOSCzdgBfDb35Lz0= github.com/Microsoft/go-winio v0.6.0 h1:slsWYD/zyx7lCXoZVlvQrj0hPTM1HI4+v1sIda2yDvg= github.com/Microsoft/go-winio v0.6.0/go.mod h1:cTAf44im0RAYeL23bpB+fzCyDH2MJiz2BO69KH/soAE= +github.com/Nordix/cluster-api/test v1.0.1-0.20241211080404-d3cdd1ac1635 h1:8IwAdDGi8ylXO0g4LWvsx49glBuHAwTPTi2U/4S6lCc= +github.com/Nordix/cluster-api/test v1.0.1-0.20241211080404-d3cdd1ac1635/go.mod h1:ieh+5ShSv+PZlfgpQz8OfUa/l7b2ApIjvxcbBqoZE08= github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 h1:wPbRQzjjwFc0ih8puEVAOFGELsn1zoIIYdxvML7mDxA= github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g= github.com/adrg/xdg v0.5.3 h1:xRnxJXne7+oWDatRhR1JLnvuccuIeCoBu2rtuLqQB78= @@ -59,8 +61,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/docker v27.3.1+incompatible h1:KttF0XoteNTicmUtBO0L2tP+J7FGRFTjaEF4k6WdhfI= -github.com/docker/docker v27.3.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v27.4.0+incompatible h1:I9z7sQ5qyzO0BfAb9IMOawRkAGxhYsidKiTMcm0DU+A= +github.com/docker/docker v27.4.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -326,8 +328,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo= -golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM= +golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= +golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE= golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -421,8 +423,6 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3 h1:2770sDpzrjjsA sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= sigs.k8s.io/cluster-api v1.9.0 h1:Iud4Zj8R/t7QX5Rvs9/V+R8HDLbf7QPVemrWfZi4g54= sigs.k8s.io/cluster-api v1.9.0/go.mod h1:8rjpkMxLFcA87Y3P6NOi6E9RMZv2uRnN9ppOPAxrTAY= -sigs.k8s.io/cluster-api/test v1.9.0 h1:0MWp8u+ihR0wFsBy/4JWVsZCcDu178B43kdP1ca6d00= -sigs.k8s.io/cluster-api/test v1.9.0/go.mod h1:w09AhrOiWTjN9ToL4xaMZLnutMFe78x03POGZ3Fsbeg= sigs.k8s.io/controller-runtime v0.19.3 h1:XO2GvC9OPftRst6xWCpTgBZO04S2cbp0Qqkj8bX1sPw= sigs.k8s.io/controller-runtime v0.19.3/go.mod h1:j4j87DqtsThvwTv5/Tc5NFRyyF/RF0ip4+62tbTSIUM= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=