From b1fc5d3b458d4777ef2d2d1db07387f88bc512be Mon Sep 17 00:00:00 2001 From: Salah Aldeen Al Saleh Date: Wed, 8 Nov 2023 17:41:41 +0000 Subject: [PATCH 1/7] Use CLI to validate strict-preflight-checks test instead of testim --- .github/workflows/build-test.yaml | 100 ++++++++++++++++++++++++------ e2e/e2e_test.go | 1 - e2e/testim/inventory/inventory.go | 9 --- 3 files changed, 82 insertions(+), 28 deletions(-) diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml index 30b03bb8af..dacda9c76e 100644 --- a/.github/workflows/build-test.yaml +++ b/.github/workflows/build-test.yaml @@ -809,7 +809,7 @@ jobs: kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}' - validate-strict-preflight-checks: + validate-strict-preflight-checks: runs-on: ubuntu-20.04 needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-mc, push-rqlite ] strategy: @@ -818,35 +818,99 @@ jobs: cluster: [ {distribution: kind, version: v1.27} ] + env: + APP_SLUG: strict-preflight-checks steps: - name: Checkout uses: actions/checkout@v3 - - name: download e2e deps - uses: actions/download-artifact@v3 + + - name: Create Cluster + id: create-cluster + uses: replicatedhq/replicated-actions/create-cluster@v1 with: - name: e2e - path: e2e/bin/ - - run: docker load -i e2e/bin/e2e-deps.tar - - run: chmod +x e2e/bin/* + api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + kubernetes-distribution: ${{ matrix.cluster.distribution }} + kubernetes-version: ${{ matrix.cluster.version }} + cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }} + timeout-minutes: '120' + ttl: 2h + export-kubeconfig: true + - name: download kots binary uses: actions/download-artifact@v3 with: name: kots path: bin/ - - run: chmod +x bin/* - - uses: ./.github/actions/kots-e2e + + - run: chmod +x bin/kots + + - name: create namespace and dockerhub secret + run: | + kubectl create ns "$APP_SLUG" + kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG" + + - name: run the test + run: | + set +e + echo ${{ secrets.STRICT_PREFLIGHT_CHECKS_LICENSE }} | base64 -d > license.yaml + ./bin/kots \ + install "$APP_SLUG/automated" \ + --license-file license.yaml \ + --no-port-forward \ + --namespace "$APP_SLUG" \ + --shared-password password \ + --kotsadm-registry ttl.sh \ + --kotsadm-namespace automated-${{ github.run_id }} \ + --skip-preflights \ + --kotsadm-tag 24h | tee output.txt + + if ! cat output.txt | grep -q 'The app was not deployed.'; then + printf "Expected to see message about app not being deployed, but did not\n" + cat output.txt + exit 1 + fi + + if ! cat output.txt | grep -q FAIL | grep -q 'The application requires a Kubernetes 2.0.0 or later.'; then + printf "Expected to see a failure about kubernetes version, but did not\n" + cat output.txt + exit 1 + fi + + # disable the strict preflight check and app should deploy successfully + ./bin/kots set config "$APP_SLUG" enable_failing_strict_analyzers="0" --namespace "$APP_SLUG" --deploy + + COUNTER=1 + while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do + ((COUNTER += 1)) + if [ $COUNTER -gt 120 ]; then + echo "Timed out waiting for app to be ready" + ./bin/kots get apps --namespace "$APP_SLUG" + echo "kotsadm logs:" + kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" + exit 1 + fi + sleep 1 + done + + printf "App is installed successfully and is ready\n\n" + ./bin/kots get apps --namespace "$APP_SLUG" + + - name: Generate support bundle on failure + if: failure() + uses: ./.github/actions/generate-support-bundle with: - test-focus: 'Strict Preflight Checks' - kots-namespace: 'strict-preflight-checks' - k8s-distribution: ${{ matrix.cluster.distribution }} - k8s-version: ${{ matrix.cluster.version }} - testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}' - testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }} + kots-namespace: "$APP_SLUG" aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' - replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}' - kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}' - kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}' + + - name: Remove Cluster + id: remove-cluster + uses: replicatedhq/replicated-actions/remove-cluster@v1 + if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }} + continue-on-error: true + with: + api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} validate-config: diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index 8a2bf453cd..9eb4877142 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -210,7 +210,6 @@ var _ = Describe("E2E", func() { Entry(nil, inventory.NewSmokeTest()), Entry(nil, inventory.NewAirgapSmokeTest()), Entry(nil, inventory.NewConfigValidation()), - Entry(nil, inventory.NewStrictPreflightChecks()), Entry(nil, inventory.NewMinimalRBACTest()), Entry(nil, inventory.NewBackupAndRestore()), Entry(nil, inventory.NewNoRequiredConfig()), diff --git a/e2e/testim/inventory/inventory.go b/e2e/testim/inventory/inventory.go index f3dea2efb1..d5f3ce917f 100644 --- a/e2e/testim/inventory/inventory.go +++ b/e2e/testim/inventory/inventory.go @@ -56,15 +56,6 @@ func NewAirgapSmokeTest() Test { } } -func NewStrictPreflightChecks() Test { - return Test{ - Name: "Strict Preflight Checks", - Suite: "strict-preflight-checks", - Namespace: "strict-preflight-checks", - UpstreamURI: "strict-preflight-checks/automated", - } -} - func NewConfigValidation() Test { return Test{ Name: "Config Validation", From aa3617be01de9507b64ee118eb728cc5b1355b39 Mon Sep 17 00:00:00 2001 From: Salah Aldeen Al Saleh Date: Wed, 8 Nov 2023 17:42:44 +0000 Subject: [PATCH 2/7] fix indentation --- .github/workflows/build-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml index dacda9c76e..6abbc355e6 100644 --- a/.github/workflows/build-test.yaml +++ b/.github/workflows/build-test.yaml @@ -809,7 +809,7 @@ jobs: kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}' - validate-strict-preflight-checks: + validate-strict-preflight-checks: runs-on: ubuntu-20.04 needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-mc, push-rqlite ] strategy: From fb6c89a266cf53e32f4d7c73a6aecd94d395b03d Mon Sep 17 00:00:00 2001 From: Salah Aldeen Al Saleh Date: Wed, 8 Nov 2023 17:55:16 +0000 Subject: [PATCH 3/7] fix lint error --- .github/workflows/build-test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml index 6abbc355e6..9a7d1635ce 100644 --- a/.github/workflows/build-test.yaml +++ b/.github/workflows/build-test.yaml @@ -864,13 +864,13 @@ jobs: --skip-preflights \ --kotsadm-tag 24h | tee output.txt - if ! cat output.txt | grep -q 'The app was not deployed.'; then + if ! grep -q 'The app was not deployed.' output.txt; then printf "Expected to see message about app not being deployed, but did not\n" cat output.txt exit 1 fi - if ! cat output.txt | grep -q FAIL | grep -q 'The application requires a Kubernetes 2.0.0 or later.'; then + if ! grep -q FAIL output.txt | grep -q 'The application requires a Kubernetes 2.0.0 or later.'; then printf "Expected to see a failure about kubernetes version, but did not\n" cat output.txt exit 1 From 83bc7172f01bb270c7e029a81d715c451de147e9 Mon Sep 17 00:00:00 2001 From: Salah Aldeen Al Saleh Date: Wed, 8 Nov 2023 17:56:41 +0000 Subject: [PATCH 4/7] move validate-strict-preflight-checks to non-testim tests --- .github/workflows/build-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml index 9a7d1635ce..ed6292dd3e 100644 --- a/.github/workflows/build-test.yaml +++ b/.github/workflows/build-test.yaml @@ -3786,7 +3786,6 @@ jobs: - validate-multi-app-backup-and-restore - validate-multi-app-install - validate-airgap-smoke-test - - validate-strict-preflight-checks - validate-config # non-testim tests - validate-minimal-rbac-override @@ -3803,6 +3802,7 @@ jobs: - validate-native-helm-v2 - validate-deployment-orchestration - validate-replicated-sdk + - validate-strict-preflight-checks # cli-only tests - validate-kots-push-images-anonymous steps: From c98356b8816bdbefdf656961226160326dbcf645 Mon Sep 17 00:00:00 2001 From: Salah Aldeen Al Saleh Date: Wed, 8 Nov 2023 18:35:49 +0000 Subject: [PATCH 5/7] fix test --- .github/workflows/build-test.yaml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml index ed6292dd3e..09e682e816 100644 --- a/.github/workflows/build-test.yaml +++ b/.github/workflows/build-test.yaml @@ -866,13 +866,18 @@ jobs: if ! grep -q 'The app was not deployed.' output.txt; then printf "Expected to see message about app not being deployed, but did not\n" - cat output.txt exit 1 fi - if ! grep -q FAIL output.txt | grep -q 'The application requires a Kubernetes 2.0.0 or later.'; then + if ! grep FAIL output.txt | grep -q 'The application requires a Kubernetes 2.0.0 or later.'; then printf "Expected to see a failure about kubernetes version, but did not\n" - cat output.txt + exit 1 + fi + + if ! kubectl logs -l app=kotsadm --namespace "$APP_SLUG" | grep -q "preflights will not be skipped, strict preflights are set to true"; then + echo "Failed to find a log line about strict preflights not being skipped in kotsadm logs" + echo "kotsadm logs:" + kubectl logs -l app=kotsadm --all-containers --namespace "$APP_SLUG" exit 1 fi From a4bb86a8ef5caa68495c9d04d83b90e70112a96e Mon Sep 17 00:00:00 2001 From: Salah Aldeen Al Saleh Date: Wed, 8 Nov 2023 18:37:34 +0000 Subject: [PATCH 6/7] updates --- .github/workflows/build-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml index 09e682e816..577cd82160 100644 --- a/.github/workflows/build-test.yaml +++ b/.github/workflows/build-test.yaml @@ -874,7 +874,7 @@ jobs: exit 1 fi - if ! kubectl logs -l app=kotsadm --namespace "$APP_SLUG" | grep -q "preflights will not be skipped, strict preflights are set to true"; then + if ! kubectl logs -l app=kotsadm -n "$APP_SLUG" | grep -q "preflights will not be skipped, strict preflights are set to true"; then echo "Failed to find a log line about strict preflights not being skipped in kotsadm logs" echo "kotsadm logs:" kubectl logs -l app=kotsadm --all-containers --namespace "$APP_SLUG" From 998e446aa0c577417ce8fafee357ecb7b3d744b5 Mon Sep 17 00:00:00 2001 From: Salah Aldeen Al Saleh Date: Wed, 8 Nov 2023 19:16:10 +0000 Subject: [PATCH 7/7] deploy/kotsadm instead of labels --- .github/workflows/build-test.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml index 577cd82160..68c4d739f6 100644 --- a/.github/workflows/build-test.yaml +++ b/.github/workflows/build-test.yaml @@ -874,10 +874,10 @@ jobs: exit 1 fi - if ! kubectl logs -l app=kotsadm -n "$APP_SLUG" | grep -q "preflights will not be skipped, strict preflights are set to true"; then + if ! kubectl logs deploy/kotsadm -n "$APP_SLUG" | grep -q "preflights will not be skipped, strict preflights are set to true"; then echo "Failed to find a log line about strict preflights not being skipped in kotsadm logs" echo "kotsadm logs:" - kubectl logs -l app=kotsadm --all-containers --namespace "$APP_SLUG" + kubectl logs deploy/kotsadm -n "$APP_SLUG" exit 1 fi @@ -891,7 +891,7 @@ jobs: echo "Timed out waiting for app to be ready" ./bin/kots get apps --namespace "$APP_SLUG" echo "kotsadm logs:" - kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" + kubectl logs deploy/kotsadm --tail=100 -n "$APP_SLUG" exit 1 fi sleep 1