Skip to content

Commit

Permalink
Add scalability test by reusing capi scale e2e test with clusterclass
Browse files Browse the repository at this point in the history
Signed-off-by: Mohammed Boukhalfa <[email protected]>
  • Loading branch information
mboukhalfa authored and mquhuy committed Dec 12, 2024
1 parent ce9c2b1 commit 70cbb85
Show file tree
Hide file tree
Showing 14 changed files with 652 additions and 40 deletions.
2 changes: 2 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,8 @@ E2E_TEMPLATES_DIR ?= $(ROOT_DIR)/test/e2e/data/infrastructure-metal3
cluster-templates: $(KUSTOMIZE) ## Generate cluster templates
$(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/cluster-template-ubuntu > $(E2E_OUT_DIR)/cluster-template-ubuntu.yaml
$(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/cluster-template-centos > $(E2E_OUT_DIR)/cluster-template-centos.yaml
$(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/cluster-template-centos-fake > $(E2E_OUT_DIR)/cluster-template-centos-fake.yaml
$(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/clusterclass-metal3 > $(E2E_OUT_DIR)/clusterclass-metal3.yaml
$(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/cluster-template-upgrade-workload > $(E2E_OUT_DIR)/cluster-template-upgrade-workload.yaml
$(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/cluster-template-centos-md-remediation > $(E2E_OUT_DIR)/cluster-template-centos-md-remediation.yaml
$(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/cluster-template-ubuntu-md-remediation > $(E2E_OUT_DIR)/cluster-template-ubuntu-md-remediation.yaml
Expand Down
12 changes: 9 additions & 3 deletions scripts/ci-e2e.sh
Original file line number Diff line number Diff line change
Expand Up @@ -34,18 +34,24 @@ export KUBERNETES_VERSION=${KUBERNETES_VERSION}
export IMAGE_OS=${IMAGE_OS}
export FORCE_REPO_UPDATE="false"
EOF
# if running a clusterctl-upgrade test skip apply bmhs in dev-env
# if running a scalability test skip apply bmhs in dev-env and run fakeIPA
if [[ ${GINKGO_FOCUS:-} == "clusterctl-upgrade" ]]; then
echo 'export SKIP_APPLY_BMH="true"' >>"${M3_DEV_ENV_PATH}/config_${USER}.sh"
fi
if [[ ${GINKGO_FOCUS:-} == "features" ]]; then
mkdir -p "$CAPI_CONFIG_FOLDER"
echo "ENABLE_BMH_NAME_BASED_PREALLOCATION: true" >"$CAPI_CONFIG_FOLDER/clusterctl.yaml"
fi
# if running a scalability test skip apply bmhs in dev-env and run fakeIPA
# if running a scalability tests, configure dev-env with fakeIPA
if [[ ${GINKGO_FOCUS:-} == "scalability" ]]; then
echo 'export SKIP_APPLY_BMH="true"' >>"${M3_DEV_ENV_PATH}/config_${USER}.sh"
echo 'export NODES_PLATFORM="fake"' >>"${M3_DEV_ENV_PATH}/config_${USER}.sh"
echo 'export SKIP_APPLY_BMH="true"' >>"${M3_DEV_ENV_PATH}/config_${USER}.sh"
mkdir -p "$CAPI_CONFIG_FOLDER"
echo 'CLUSTER_TOPOLOGY: true' >"$CAPI_CONFIG_FOLDER/clusterctl.yaml"
echo 'export EPHEMERAL_CLUSTER="minikube"' >>"${M3_DEV_ENV_PATH}/config_${USER}.sh"
else
# Don't run scalability tests if not asked for.
export GINKGO_SKIP="${GINKGO_SKIP:-} scalability"
fi
# Run make devenv to boot the source cluster
pushd "${M3_DEV_ENV_PATH}" || exit 1
Expand Down
7 changes: 5 additions & 2 deletions scripts/environment.sh
Original file line number Diff line number Diff line change
Expand Up @@ -64,8 +64,11 @@ fi

# Scalability test environment vars and config
if [[ ${GINKGO_FOCUS:-} == "scalability" ]]; then
export NUM_NODES=${NUM_NODES:-"100"}
export BMH_BATCH_SIZE=${BMH_BATCH_SIZE:-"20"}
export NUM_NODES=${NUM_NODES:-"10"}
export BMH_BATCH_SIZE=${BMH_BATCH_SIZE:-"2"}
export CONTROL_PLANE_MACHINE_COUNT=${CONTROL_PLANE_MACHINE_COUNT:-"1"}
export WORKER_MACHINE_COUNT=${WORKER_MACHINE_COUNT:-"0"}
export KUBERNETES_VERSION_UPGRADE_FROM=${FROM_K8S_VERSION}
fi

# Integration test environment vars and config
Expand Down
8 changes: 7 additions & 1 deletion test/e2e/config/e2e_conf.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -188,8 +188,13 @@ providers:
targetName: "cluster-template-ubuntu.yaml"
- sourcePath: "../_out/cluster-template-centos.yaml"
targetName: "cluster-template-centos.yaml"
- sourcePath: "../_out/cluster-template-centos-fake.yaml"
targetName: "cluster-template-centos-fake.yaml"
- sourcePath: "../_out/cluster-template-centos-fake.yaml"
targetName: "cluster-template-ubuntu-fake.yaml"
- sourcePath: "../_out/clusterclass.yaml"
targetName: "clusterclass-test-clusterclass.yaml"
- sourcePath: "../_out/clusterclass-metal3.yaml"
- sourcePath: "../_out/cluster-template-centos-md-remediation.yaml"
targetName: "cluster-template-centos-md-remediation.yaml"
- sourcePath: "../_out/cluster-template-ubuntu-md-remediation.yaml"
Expand Down Expand Up @@ -246,7 +251,8 @@ variables:
BMO_RELEASE_0.6: "data/bmo-deployment/overlays/release-0.6"
BMO_RELEASE_0.8: "data/bmo-deployment/overlays/release-0.8"
BMO_RELEASE_LATEST: "data/bmo-deployment/overlays/release-latest"

FKAS_RELEASE_LATEST: "data/fkas"

intervals:
default/wait-controllers: ["10m", "10s"]
default/wait-cluster: ["20m", "30s"] # The second time to check the availibility of the cluster should happen late, so kcp object has time to be created
Expand Down
2 changes: 2 additions & 0 deletions test/e2e/data/fkas/kustomization.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
resources:
- resources.yaml
77 changes: 77 additions & 0 deletions test/e2e/data/fkas/resources.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
---
apiVersion: v1
kind: Namespace
metadata:
name: fkas-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: metal3-fkas-sa
namespace: fkas-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: metal3-fkas-role
rules:
- apiGroups: ["metal3.io", "infrastructure.cluster.x-k8s.io"]
resources: ["baremetalhosts", "metal3machines"]
verbs: ["get", "list", "watch"]
- apiGroups: ["cluster.x-k8s.io"]
resources: ["machines"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get", "list"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: metal3-fkas-rolebinding
subjects:
- kind: ServiceAccount
name: metal3-fkas-sa
namespace: fkas-system
roleRef:
kind: ClusterRole
name: metal3-fkas-role
apiGroup: rbac.authorization.k8s.io
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: metal3-fkas-system
namespace: fkas-system
spec:
replicas: 1
selector:
matchLabels:
app: metal3-fkas-system
template:
metadata:
labels:
app: metal3-fkas-system
spec:
serviceAccountName: metal3-fkas-sa
hostNetwork: true
containers:
- name: metal3-fkas-reconciler
image: quay.io/metal3-io/metal3-fkas:latest
imagePullPolicy: IfNotPresent
command: ["/reconciler"]
env:
- name: DEBUG
value: "true"
- image: quay.io/metal3-io/metal3-fkas:latest
imagePullPolicy: IfNotPresent
ports:
- containerPort: 3333
env:
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: DEBUG
value: "true"
name: metal3-fkas
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
---
apiVersion: cluster.x-k8s.io/v1beta1
kind: Cluster
metadata:
labels:
cni: ${CLUSTER_NAME}-crs-0
name: ${CLUSTER_NAME}
namespace: ${NAMESPACE}
spec:
clusterNetwork:
pods:
cidrBlocks: ["${POD_CIDR}"]
services:
cidrBlocks: ["${SERVICE_CIDR}"]
topology:
class: metal3
version: ${KUBERNETES_VERSION}
controlPlane:
replicas: ${CONTROL_PLANE_MACHINE_COUNT}
workers:
machineDeployments:
- class: worker
name: ${CLUSTER_NAME}-machine
replicas: ${WORKER_MACHINE_COUNT}
variables:
- name: image
value:
checksum: ${IMAGE_RAW_CHECKSUM}
checksumType: ${IMAGE_CHECKSUM_TYPE}
format: raw
url: ${IMAGE_RAW_URL}
- name: controlPlaneEndpoint
value:
host: CLUSTER_APIENDPOINT_HOST_HOLDER
port: CLUSTER_APIENDPOINT_PORT_HOLDER
- name: controlPlaneDataTemplate
value: ${CLUSTER_NAME}-controlplane-template
---
apiVersion: ipam.metal3.io/v1alpha1
kind: IPPool
metadata:
name: ${CLUSTER_NAME}-baremetalv4-pool
spec:
clusterName: ${CLUSTER_NAME}
gateway: 192.168.111.1
namePrefix: ${CLUSTER_NAME}-bmv4
pools:
- end: 192.168.111.240
start: 192.168.111.201
prefix: 24
---
apiVersion: ipam.metal3.io/v1alpha1
kind: IPPool
metadata:
name: ${CLUSTER_NAME}-provisioning-pool
spec:
clusterName: ${CLUSTER_NAME}
namePrefix: ${CLUSTER_NAME}-prov
pools:
- end: 172.22.0.240
start: 172.22.0.201
prefix: 24
---
apiVersion: infrastructure.cluster.x-k8s.io/${CAPM3_VERSION}
kind: Metal3DataTemplate
metadata:
name: ${CLUSTER_NAME}-controlplane-template
spec:
clusterName: ${CLUSTER_NAME}
metaData:
ipAddressesFromIPPool:
- key: provisioningIP
name: ${CLUSTER_NAME}-provisioning-pool
objectNames:
- key: name
object: machine
- key: local-hostname
object: machine
- key: local_hostname
object: machine
prefixesFromIPPool:
- key: provisioningCIDR
name: ${CLUSTER_NAME}-provisioning-pool
networkData:
links:
ethernets:
- id: enp1s0
macAddress:
fromHostInterface: enp1s0
type: phy
- id: enp2s0
macAddress:
fromHostInterface: enp2s0
type: phy
networks:
ipv4:
- id: baremetalv4
ipAddressFromIPPool: ${CLUSTER_NAME}-baremetalv4-pool
link: enp2s0
routes:
- gateway:
fromIPPool: ${CLUSTER_NAME}-baremetalv4-pool
network: 0.0.0.0
prefix: 0
services:
dns:
- 8.8.8.8
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
resources:
- cluster-with-topology.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
resources:
- ../bases/cluster-with-topology
Loading

0 comments on commit 70cbb85

Please sign in to comment.