fix: ensure only argo jobs are run on nodes created by karpenter TDE-…

…1185 (#634)

#### Motivation

We would prefer only short term jobs from argo are run on large nodes
that are created by karpenter

#### Modification

taint all provisoners that karpenter has so pods have to opt into being
created by karpenter.

#### Checklist

_If not applicable, provide explanation of why._

- [ ] Tests updated
- [ ] Docs updated
- [ ] Issue linked in Title

infra/charts/karpenter.ts

@@ -142,6 +142,11 @@ export class KarpenterProvisioner extends Chart {
     const provisionAmd64OnDemand = new Provisioner(this, 'ClusterAmd64WorkerNodesOnDemand', {
       metadata: { name: `karpenter-amd64-on-demand`, namespace: 'karpenter' },
       spec: {
+        taints: [
+          // Ensure only pods that tolerate karpenter's capacity run on this node
+          // to prevent long running pods (eg kube-dns) being moved.
+          { key: 'karpenter.sh/capacity-type', value: 'on-demand', effect: 'NoSchedule' },
+        ],
         requirements: [
           { key: 'karpenter.sh/capacity-type', operator: 'In', values: ['on-demand'] },
           { key: 'kubernetes.io/arch', operator: 'In', values: ['amd64'] },

workflows/test/sleep.yml

@@ -11,7 +11,12 @@ spec:
   templates:
     - name: sleep
       nodeSelector:
-        karpenter.sh/capacity-type: 'spot'
+        karpenter.sh/capacity-type: 'on-demand'
+      tolerations:
+        - key: 'karpenter.sh/capacity-type'
+          operator: 'Equal'
+          value: 'on-demand'
+          effect: 'NoSchedule'
       container:
         resources:
           requests: