Skip to content

Commit

Permalink
Add karpenter workflow
Browse files Browse the repository at this point in the history 
Signed-off-by: Heba Elayoty <[email protected]>
  • Loading branch information
@helayoty
helayoty committed May 8, 2024
1 parent cc8e02d commit 3f79420
Show file tree
Hide file tree
Showing 4 changed files with 377 additions and 79 deletions.
4 changes: 4 additions & 0 deletions .github/workflows/e2e-workflow.yml
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
name: kaito-e2e-workflow

concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true

on:
workflow_call:
inputs:
Expand Down
18 changes: 13 additions & 5 deletions .github/workflows/kaito-e2e.yml
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,5 @@
name: pr-e2e-test

concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true

on:
pull_request:
paths-ignore: ['docs/**', '**.md', '**.mdx', '**.png', '**.jpg']
Expand All @@ -16,7 +12,7 @@ permissions:
contents: read # This is required for actions/checkout

jobs:
run-e2e:
run-kaito-gpu-provisioner-e2e:
uses: ./.github/workflows/e2e-workflow.yml
with:
git_sha: ${{ github.event.pull_request.head.sha }}
Expand All @@ -27,3 +23,15 @@ jobs:
E2E_AMRT_SECRET_NAME: ${{ secrets.AMRT_SECRET_NAME }}
E2E_ACR_AMRT_USERNAME: ${{ secrets.ACR_AMRT_USERNAME }}
E2E_ACR_AMRT_PASSWORD: ${{ secrets.ACR_AMRT_PASSWORD }}

run-kaito-karpenter-e2e:
uses: ./.github/workflows/karpenter-e2e-workflow.yml
with:
git_sha: ${{ github.event.pull_request.head.sha }}
secrets:
E2E_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
E2E_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
E2E_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
E2E_AMRT_SECRET_NAME: ${{ secrets.AMRT_SECRET_NAME }}
E2E_ACR_AMRT_USERNAME: ${{ secrets.ACR_AMRT_USERNAME }}
E2E_ACR_AMRT_PASSWORD: ${{ secrets.ACR_AMRT_PASSWORD }}
215 changes: 215 additions & 0 deletions .github/workflows/karpenter-e2e-workflow.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,215 @@
name: kaito-karpenter-e2e-workflow

concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true

on:
workflow_call:
inputs:
git_sha:
type: string
required: true
tag:
type: string
isRelease:
type: boolean
default: false
registry:
type: string
region:
type: string
description: "the azure location to run the e2e test in"
default: "eastus"
k8s_version:
type: string
default: "1.29.2"
secrets:
E2E_CLIENT_ID:
required: true
E2E_TENANT_ID:
required: true
E2E_SUBSCRIPTION_ID:
required: true
E2E_AMRT_SECRET_NAME:
required: true
E2E_ACR_AMRT_USERNAME:
required: true
E2E_ACR_AMRT_PASSWORD:
required: true

permissions:
contents: read # This is required for actions/checkout

jobs:
azure-e2e-tests:
runs-on: ubuntu-latest
permissions:
contents: read
id-token: write # This is required for requesting the JWT
environment: e2e-test
env:
GO_VERSION: "1.22"

steps:
- name: Harden Runner
uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0
with:
egress-policy: audit

- name: Checkout
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
ref: ${{ inputs.git_sha }}

- name: Set e2e Resource and Cluster Name
run: |
rand=$(git rev-parse --short ${{ inputs.git_sha }})
if [ "$rand" = "" ]; then
rand=$RANDOM
fi
echo "VERSION=${rand}" >> $GITHUB_ENV
echo "CLUSTER_NAME=kaito${rand}" >> $GITHUB_ENV
echo "RUN_LLAMA_13B=false" >> $GITHUB_ENV
echo "REGISTRY=kaito${rand}.azurecr.io" >> $GITHUB_ENV
- name: Set Registry
if: ${{ inputs.isRelease }}
run: |
echo "REGISTRY=${{ inputs.registry }}" >> $GITHUB_ENV
echo "VERSION=$(echo ${{ inputs.tag }} | tr -d v)" >> $GITHUB_ENV
- name: Set up Go ${{ env.GO_VERSION }}
uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}

- name: Az login
uses: azure/login@8c334a195cbb38e46038007b304988d888bf676a # v2.0.0
with:
client-id: ${{ secrets.E2E_CLIENT_ID }}
tenant-id: ${{ secrets.E2E_TENANT_ID }}
subscription-id: ${{ secrets.E2E_SUBSCRIPTION_ID }}

- uses: azure/setup-helm@v4
with:
token: ${{ secrets.GITHUB_TOKEN }}
id: install

- name: Create Resource Group
shell: bash
run: |
make create-rg
env:
AZURE_RESOURCE_GROUP: ${{ env.CLUSTER_NAME }}

- name: Create ACR
shell: bash
run: |
make create-acr
env:
AZURE_RESOURCE_GROUP: ${{ env.CLUSTER_NAME }}
AZURE_ACR_NAME: ${{ env.CLUSTER_NAME }}

- name: Create Karpenter Azure Identity
uses: azure/[email protected]
with:
inlineScript: |
az identity create --name karpentermsi --resource-group ${{ env.CLUSTER_NAME }}
- name: build KAITO image
if: ${{ !inputs.isRelease }}
shell: bash
run: |
make docker-build-kaito
env:
REGISTRY: ${{ env.REGISTRY }}
VERSION: ${{ env.VERSION }}

- name: create cluster
shell: bash
run: |
make create-aks-cluster-with-kaito
env:
AZURE_ACR_NAME: ${{ env.CLUSTER_NAME }}
AZURE_RESOURCE_GROUP: ${{ env.CLUSTER_NAME }}
AZURE_CLUSTER_NAME: ${{ env.CLUSTER_NAME }}
AZURE_LOCATION: ${{ inputs.region }}
AKS_K8S_VERSION: ${{ inputs.k8s_version }}

- name: Install karpenter Azure provider helm chart
shell: bash
run: |
make azure-karpenter-helm
kubectl wait --for=condition=available deploy "karpenter" -n karpenter --timeout=300s
env:
AZURE_RESOURCE_GROUP: ${{ env.CLUSTER_NAME }}
AZURE_CLUSTER_NAME: ${{ env.CLUSTER_NAME }}
AZURE_TENANT_ID: ${{ secrets.E2E_TENANT_ID }}
AZURE_SUBSCRIPTION_ID: ${{ secrets.E2E_SUBSCRIPTION_ID }}
KARPENTER_VERSION: ${{ vars.KARPENTER_VERSION }}

- uses: azure/login@8c334a195cbb38e46038007b304988d888bf676a # v2.0.0
with:
client-id: ${{ secrets.E2E_CLIENT_ID }}
tenant-id: ${{ secrets.E2E_TENANT_ID }}
subscription-id: ${{ secrets.E2E_SUBSCRIPTION_ID }}

- name: Create Role Assignment
uses: azure/[email protected]
with:
inlineScript: |
KARPENTER_USER_ASSIGNED_PRINCIPAL_ID="$(az identity show --name karpentermsi --resource-group ${{ env.CLUSTER_NAME }} --query 'principalId' -otsv)"
for role in "Virtual Machine Contributor" "Network Contributor" "Managed Identity Operator"; do \
az role assignment create --assignee "${KARPENTER_USER_ASSIGNED_PRINCIPAL_ID}" \
--scope "/subscriptions/${{ secrets.E2E_SUBSCRIPTION_ID }}/resourceGroups/${{ env.CLUSTER_NAME }}" --role "$role"
done
- name: Create Azure Federated Identity
uses: azure/[email protected]
with:
inlineScript: |
AKS_OIDC_ISSUER="$(az aks show -n "${{ env.CLUSTER_NAME }}" -g "${{ env.CLUSTER_NAME }}" --query 'oidcIssuerProfile.issuerUrl' -otsv)"
az identity federated-credential create --name karpenter-fed --identity-name karpentermsi --resource-group "${{ env.CLUSTER_NAME }}" \
--issuer "${AKS_OIDC_ISSUER}" --subject system:serviceaccount:"karpenter:karpenter-sa" --audience api://AzureADTokenExchange
- name: Install KAITO Workspace helm chart
shell: bash
run: |
make az-patch-install-helm
kubectl wait --for=condition=available deploy "kaito-workspace" -n kaito-workspace --timeout=300s
env:
AZURE_RESOURCE_GROUP: ${{ env.CLUSTER_NAME }}
AZURE_CLUSTER_NAME: ${{ env.CLUSTER_NAME }}
REGISTRY: ${{ env.REGISTRY }}
VERSION: ${{ env.VERSION }}

- name: Add Secret Credentials
run: |
kubectl create secret docker-registry ${{ secrets.E2E_AMRT_SECRET_NAME }} \
--docker-server=${{ secrets.E2E_ACR_AMRT_USERNAME }}.azurecr.io \
--docker-username=${{ secrets.E2E_ACR_AMRT_USERNAME }} \
--docker-password=${{ secrets.E2E_ACR_AMRT_PASSWORD }}
- name: Log kaito-workspace
run: |
kubectl get pods -n kaito-workspace -o name | grep "^pod/kaito-workspace" | sed 's/^pod\///' | xargs -I {} kubectl logs -n kaito-workspace {}
- name: Run e2e test
run: |
make kaito-karpenter-e2e-test
env:
AZURE_CLUSTER_NAME: ${{ env.CLUSTER_NAME }}
RUN_LLAMA_13B: ${{ env.RUN_LLAMA_13B }}
AI_MODELS_REGISTRY: ${{ secrets.E2E_ACR_AMRT_USERNAME }}.azurecr.io
AI_MODELS_REGISTRY_SECRET: ${{ secrets.E2E_AMRT_SECRET_NAME }}

- name: Cleanup e2e resources
if: ${{ always() }}
uses: azure/[email protected]
with:
inlineScript: |
set +e
az group delete --name "${{ env.CLUSTER_NAME }}" --yes --no-wait || true
Loading

0 comments on commit 3f79420

Please sign in to comment.