Add karpenter workflow

Signed-off-by: Heba Elayoty <[email protected]>
kaito-project · May 9, 2024 · c75f90b · c75f90b
1 parent cc8e02d
commit c75f90b
Show file tree

Hide file tree

Showing 3 changed files with 200 additions and 102 deletions.
diff --git a/.github/workflows/e2e-workflow.yml b/.github/workflows/e2e-workflow.yml
@@ -6,6 +6,9 @@ on:
       git_sha:
         type: string
         required: true
+      suite:
+        type: string
+        required: true
       tag:
         type: string
       isRelease:
@@ -34,12 +37,10 @@ on:
       E2E_ACR_AMRT_PASSWORD:
         required: true
 
-permissions:
-  contents: read # This is required for actions/checkout
-
 jobs:
   e2e-tests:
     runs-on: ubuntu-latest
+    name: e2e-tests-${{ inputs.suite }}
     permissions:
       contents: read
       id-token: write # This is required for requesting the JWT
@@ -67,15 +68,15 @@ jobs:
           fi
 
           echo "VERSION=${rand}" >> $GITHUB_ENV
-          echo "CLUSTER_NAME=kaito${rand}" >> $GITHUB_ENV
+          echo "CLUSTER_NAME=${{ inputs.suite }}{rand}" >> $GITHUB_ENV
+          echo "REGISTRY=${{ inputs.suite }}{rand}.azurecr.io" >> $GITHUB_ENV
           echo "RUN_LLAMA_13B=false" >> $GITHUB_ENV
-          echo "REGISTRY=kaito${rand}.azurecr.io" >> $GITHUB_ENV
 
       - name: Set Registry
         if: ${{ inputs.isRelease }}
         run: |
-              echo "REGISTRY=${{ inputs.registry }}" >> $GITHUB_ENV
-              echo "VERSION=$(echo ${{ inputs.tag }} | tr -d v)" >> $GITHUB_ENV
+          echo "REGISTRY=${{ inputs.registry }}" >> $GITHUB_ENV
+          echo "VERSION=$(echo ${{ inputs.tag }} | tr -d v)" >> $GITHUB_ENV
 
       - name: Set up Go ${{ env.GO_VERSION }}
         uses: actions/setup-go@v5
@@ -113,7 +114,7 @@ jobs:
         uses: azure/[email protected]
         with:
           inlineScript: |
-            az identity create --name gpuIdentity --resource-group  ${{ env.CLUSTER_NAME }}
+            az identity create --name ${{ inputs.suite }}Identity --resource-group  ${{ env.CLUSTER_NAME }}
 
       - name: build KAITO image
         if: ${{ !inputs.isRelease }}
@@ -136,6 +137,7 @@ jobs:
           AKS_K8S_VERSION: ${{ inputs.k8s_version }}
 
       - name: Install gpu-provisioner helm chart
+        if: ${{ inputs.suite == 'gpuprov' }}
         shell: bash
         run: |
           make gpu-provisioner-helm
@@ -147,6 +149,21 @@ jobs:
           AZURE_SUBSCRIPTION_ID: ${{ secrets.E2E_SUBSCRIPTION_ID }}
           GPU_PROVISIONER_VERSION: ${{ vars.GPU_PROVISIONER_VERSION }}
 
+      - name: Install karpenter Azure provider helm chart
+        if: ${{ inputs.suite == 'azkarpenter' }}
+        shell: bash
+        run: |
+          make azure-karpenter-helm
+          # taint nodes as karpenter-system
+          kubectl taint nodes CriticalAddonsOnly=true:NoSchedule --all
+          kubectl wait --for=condition=available deploy "karpenter" -n karpenter --timeout=300s
+        env:
+          AZURE_RESOURCE_GROUP: ${{ env.CLUSTER_NAME }}
+          AZURE_CLUSTER_NAME: ${{ env.CLUSTER_NAME }}
+          AZURE_TENANT_ID: ${{ secrets.E2E_TENANT_ID }}
+          AZURE_SUBSCRIPTION_ID: ${{ secrets.E2E_SUBSCRIPTION_ID }}
+          KARPENTER_VERSION: ${{ vars.KARPENTER_VERSION }}
+
       - uses: azure/login@8c334a195cbb38e46038007b304988d888bf676a # v2.0.0
         with:
           client-id: ${{ secrets.E2E_CLIENT_ID }}
@@ -157,16 +174,25 @@ jobs:
         uses: azure/[email protected]
         with:
           inlineScript: |
-            IDENTITY_PRINCIPAL_ID="$(az identity show --name gpuIdentity --resource-group ${{ env.CLUSTER_NAME }} --query 'principalId' -otsv)"
+            IDENTITY_PRINCIPAL_ID="$(az identity show --name ${{ inputs.suite }}Identity --resource-group ${{ env.CLUSTER_NAME }} --query 'principalId' -otsv)"
             az role assignment create --assignee ${IDENTITY_PRINCIPAL_ID} --scope "/subscriptions/${{ secrets.E2E_SUBSCRIPTION_ID }}/resourceGroups/${{ env.CLUSTER_NAME }}"  --role "Contributor"
-
+            if [ "${{ inputs.suite }}" == "azkarpenter" ]; then
+              
+            fi
       - name: Create Azure Federated Identity
         uses: azure/[email protected]
         with:
           inlineScript: |
             AKS_OIDC_ISSUER="$(az aks show -n "${{ env.CLUSTER_NAME }}" -g "${{ env.CLUSTER_NAME }}" --query 'oidcIssuerProfile.issuerUrl' -otsv)"
-            az identity federated-credential create --name gpu-fed-credential --identity-name gpuIdentity --resource-group "${{ env.CLUSTER_NAME }}" \
-            --issuer "${AKS_OIDC_ISSUER}" --subject system:serviceaccount:"gpu-provisioner:gpu-provisioner" --audience api://AzureADTokenExchange
+            
+            if [ "${{ inputs.suite }}" == "gpuprov" ]; then
+              az identity federated-credential create --name ${{ inputs.suite }}-fed --identity-name ${{ inputs.suite }}Identity --resource-group "${{ env.CLUSTER_NAME }}" \
+              --issuer "${AKS_OIDC_ISSUER}" --subject system:serviceaccount:"gpu-provisioner:gpu-provisioner" --audience api://AzureADTokenExchange
+            fi
+            if [ "${{ inputs.suite }}" == "azkarpenter" ]; then
+              az identity federated-credential create --name ${{ inputs.suite }}-fed --identity-name ${{ inputs.suite }}Identity --resource-group "${{ env.CLUSTER_NAME }}" \
+              --issuer "${AKS_OIDC_ISSUER}" --subject system:serviceaccount:"karpenter:karpenter-sa" --audience api://AzureADTokenExchange
+            fi
 
       - name: Install KAITO Workspace helm chart
         shell: bash

diff --git a/.github/workflows/kaito-e2e.yml b/.github/workflows/kaito-e2e.yml
@@ -1,9 +1,5 @@
 name: pr-e2e-test
 
-concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
-  cancel-in-progress: true
-
 on:
   pull_request:
     paths-ignore: ['docs/**', '**.md', '**.mdx', '**.png', '**.jpg']
@@ -17,13 +13,22 @@ permissions:
 
 jobs:
   run-e2e:
-    uses: ./.github/workflows/e2e-workflow.yml
-    with:
-      git_sha: ${{ github.event.pull_request.head.sha }}
-    secrets:
-      E2E_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
-      E2E_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
-      E2E_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
-      E2E_AMRT_SECRET_NAME: ${{ secrets.AMRT_SECRET_NAME }}
-      E2E_ACR_AMRT_USERNAME: ${{ secrets.ACR_AMRT_USERNAME }}
-      E2E_ACR_AMRT_PASSWORD: ${{ secrets.ACR_AMRT_PASSWORD }}
+  strategy:
+    fail-fast: false
+    matrix:
+      suite: [ gpuprov, azkarpenter ]
+  permissions:
+    contents: read
+    id-token: write
+    statuses: write
+  uses: ./.github/workflows/e2e-workflow.yml
+  with:
+    git_sha: ${{ github.event.pull_request.head.sha }}
+    suite: ${{ matrix.suite }}
+  secrets:
+    E2E_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
+    E2E_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
+    E2E_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+    E2E_AMRT_SECRET_NAME: ${{ secrets.AMRT_SECRET_NAME }}
+    E2E_ACR_AMRT_USERNAME: ${{ secrets.ACR_AMRT_USERNAME }}
+    E2E_ACR_AMRT_PASSWORD: ${{ secrets.ACR_AMRT_PASSWORD }}