From cf8faca1fccfd5e7088c9ed69a94721eb7bb705f Mon Sep 17 00:00:00 2001
From: Blayne Chard <bchard@linz.govt.nz>
Date: Fri, 21 Jun 2024 09:18:14 +1200
Subject: [PATCH] fix: ensure only argo jobs are run on nodes created by
 karpenter TDE-1185 (#634)

#### Motivation

We would prefer only short term jobs from argo are run on large nodes
that are created by karpenter

#### Modification

taint all provisoners that karpenter has so pods have to opt into being
created by karpenter.

#### Checklist

_If not applicable, provide explanation of why._

- [ ] Tests updated
- [ ] Docs updated
- [ ] Issue linked in Title
---
 infra/charts/karpenter.ts | 5 +++++
 workflows/test/sleep.yml  | 7 ++++++-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/infra/charts/karpenter.ts b/infra/charts/karpenter.ts
index 616b62104..fcd838af3 100644
--- a/infra/charts/karpenter.ts
+++ b/infra/charts/karpenter.ts
@@ -142,6 +142,11 @@ export class KarpenterProvisioner extends Chart {
     const provisionAmd64OnDemand = new Provisioner(this, 'ClusterAmd64WorkerNodesOnDemand', {
       metadata: { name: `karpenter-amd64-on-demand`, namespace: 'karpenter' },
       spec: {
+        taints: [
+          // Ensure only pods that tolerate karpenter's capacity run on this node
+          // to prevent long running pods (eg kube-dns) being moved.
+          { key: 'karpenter.sh/capacity-type', value: 'on-demand', effect: 'NoSchedule' },
+        ],
         requirements: [
           { key: 'karpenter.sh/capacity-type', operator: 'In', values: ['on-demand'] },
           { key: 'kubernetes.io/arch', operator: 'In', values: ['amd64'] },
diff --git a/workflows/test/sleep.yml b/workflows/test/sleep.yml
index a8b3a537a..f45a8a881 100644
--- a/workflows/test/sleep.yml
+++ b/workflows/test/sleep.yml
@@ -11,7 +11,12 @@ spec:
   templates:
     - name: sleep
       nodeSelector:
-        karpenter.sh/capacity-type: 'spot'
+        karpenter.sh/capacity-type: 'on-demand'
+      tolerations:
+        - key: 'karpenter.sh/capacity-type'
+          operator: 'Equal'
+          value: 'on-demand'
+          effect: 'NoSchedule'
       container:
         resources:
           requests: