From ff44f7325bf006fecf66e2a5684a7d8700e51cfd Mon Sep 17 00:00:00 2001 From: Amanuel Engeda <74629455+engedaam@users.noreply.github.com> Date: Fri, 17 Jan 2025 15:58:35 -0800 Subject: [PATCH 01/17] fix: Start E2E montitioning controllers at the start of the test suite (#1878) --- test/pkg/debug/setup.go | 18 +++++++++++++----- test/suites/perf/suite_test.go | 2 ++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/test/pkg/debug/setup.go b/test/pkg/debug/setup.go index 0329029195..5867d2df2e 100644 --- a/test/pkg/debug/setup.go +++ b/test/pkg/debug/setup.go @@ -34,12 +34,20 @@ const ( var m *Monitor var e *EventClient +func BeforeSuite(ctx context.Context, config *rest.Config, kubeClient client.Client) { + m = New(ctx, config, kubeClient) + m.MustStart() +} + +func AfterSuite(ctx context.Context, config *rest.Config, kubeClient client.Client) { + m.Stop() +} + func BeforeEach(ctx context.Context, config *rest.Config, kubeClient client.Client) { // If the test is labeled as NoWatch, then the node/pod monitor will just list at the beginning // of the test rather than perform a watch during it - if !lo.Contains(ginkgo.CurrentSpecReport().Labels(), NoWatch) { - m = New(ctx, config, kubeClient) - m.MustStart() + if lo.Contains(ginkgo.CurrentSpecReport().Labels(), NoWatch) { + m.Stop() } if !lo.Contains(ginkgo.CurrentSpecReport().Labels(), NoEvents) { e = NewEventClient(kubeClient) @@ -47,8 +55,8 @@ func BeforeEach(ctx context.Context, config *rest.Config, kubeClient client.Clie } func AfterEach(ctx context.Context) { - if !lo.Contains(ginkgo.CurrentSpecReport().Labels(), NoWatch) { - m.Stop() + if lo.Contains(ginkgo.CurrentSpecReport().Labels(), NoWatch) { + m.MustStart() } if !lo.Contains(ginkgo.CurrentSpecReport().Labels(), NoEvents) { Expect(e.DumpEvents(ctx)).To(Succeed()) diff --git a/test/suites/perf/suite_test.go b/test/suites/perf/suite_test.go index ecc1b0ea7e..0d150ca0c5 100644 --- a/test/suites/perf/suite_test.go +++ b/test/suites/perf/suite_test.go @@ -46,12 +46,14 @@ func TestPerf(t *testing.T) { RegisterFailHandler(Fail) BeforeSuite(func() { env = common.NewEnvironment(t) + debug.BeforeSuite(env.Context, env.Config, env.Client) }) AfterSuite(func() { // Write out the timestamps from our tests if err := debug.WriteTimestamps(env.OutputDir, env.TimeIntervalCollector); err != nil { log.FromContext(env).Info(fmt.Sprintf("Failed to write timestamps to files, %s", err)) } + debug.AfterSuite(env.Context, env.Config, env.Client) env.Stop() }) RunSpecs(t, "Perf") From c10940d5bb09ef7eb01040182a68e192b6652a63 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jan 2025 21:06:36 -0800 Subject: [PATCH 02/17] chore(deps): bump the k8s-go-deps group with 8 updates (#1923) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 22 +++++++++++++--------- go.sum | 36 ++++++++++++++++++++---------------- 2 files changed, 33 insertions(+), 25 deletions(-) diff --git a/go.mod b/go.mod index de77702ded..259364fd36 100644 --- a/go.mod +++ b/go.mod @@ -21,16 +21,16 @@ require ( go.uber.org/zap v1.27.0 golang.org/x/text v0.21.0 golang.org/x/time v0.9.0 - k8s.io/api v0.32.0 - k8s.io/apiextensions-apiserver v0.32.0 - k8s.io/apimachinery v0.32.0 - k8s.io/client-go v0.32.0 - k8s.io/cloud-provider v0.32.0 - k8s.io/component-base v0.32.0 - k8s.io/csi-translation-lib v0.32.0 + k8s.io/api v0.32.1 + k8s.io/apiextensions-apiserver v0.32.1 + k8s.io/apimachinery v0.32.1 + k8s.io/client-go v0.32.1 + k8s.io/cloud-provider v0.32.1 + k8s.io/component-base v0.32.1 + k8s.io/csi-translation-lib v0.32.1 k8s.io/klog/v2 v2.130.1 k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 - sigs.k8s.io/controller-runtime v0.19.4 + sigs.k8s.io/controller-runtime v0.20.0 ) require ( @@ -83,7 +83,11 @@ require ( sigs.k8s.io/yaml v1.4.0 // indirect ) -require github.com/fsnotify/fsnotify v1.7.0 // indirect +require ( + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/google/btree v1.1.3 // indirect + golang.org/x/sync v0.10.0 // indirect +) retract ( v0.100.101-test // accidentally published testing version diff --git a/go.sum b/go.sum index 17fc76e448..aed8014d0f 100644 --- a/go.sum +++ b/go.sum @@ -44,6 +44,8 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= +github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -155,6 +157,8 @@ golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbht golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -192,28 +196,28 @@ gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.32.0 h1:OL9JpbvAU5ny9ga2fb24X8H6xQlVp+aJMFlgtQjR9CE= -k8s.io/api v0.32.0/go.mod h1:4LEwHZEf6Q/cG96F3dqR965sYOfmPM7rq81BLgsE0p0= -k8s.io/apiextensions-apiserver v0.32.0 h1:S0Xlqt51qzzqjKPxfgX1xh4HBZE+p8KKBq+k2SWNOE0= -k8s.io/apiextensions-apiserver v0.32.0/go.mod h1:86hblMvN5yxMvZrZFX2OhIHAuFIMJIZ19bTvzkP+Fmw= -k8s.io/apimachinery v0.32.0 h1:cFSE7N3rmEEtv4ei5X6DaJPHHX0C+upp+v5lVPiEwpg= -k8s.io/apimachinery v0.32.0/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= -k8s.io/client-go v0.32.0 h1:DimtMcnN/JIKZcrSrstiwvvZvLjG0aSxy8PxN8IChp8= -k8s.io/client-go v0.32.0/go.mod h1:boDWvdM1Drk4NJj/VddSLnx59X3OPgwrOo0vGbtq9+8= -k8s.io/cloud-provider v0.32.0 h1:QXYJGmwME2q2rprymbmw2GroMChQYc/MWN6l/I4Kgp8= -k8s.io/cloud-provider v0.32.0/go.mod h1:cz3gVodkhgwi2ugj/JUPglIruLSdDaThxawuDyCHfr8= -k8s.io/component-base v0.32.0 h1:d6cWHZkCiiep41ObYQS6IcgzOUQUNpywm39KVYaUqzU= -k8s.io/component-base v0.32.0/go.mod h1:JLG2W5TUxUu5uDyKiH2R/7NnxJo1HlPoRIIbVLkK5eM= -k8s.io/csi-translation-lib v0.32.0 h1:RAn9RGgYXHJQtDSb6qQ7zvq6QObOejzmsXDARI+f4OQ= -k8s.io/csi-translation-lib v0.32.0/go.mod h1:TjCJzkTNstdOESAXNnEImrYOMIEzP14aqM7H+vkehqw= +k8s.io/api v0.32.1 h1:f562zw9cy+GvXzXf0CKlVQ7yHJVYzLfL6JAS4kOAaOc= +k8s.io/api v0.32.1/go.mod h1:/Yi/BqkuueW1BgpoePYBRdDYfjPF5sgTr5+YqDZra5k= +k8s.io/apiextensions-apiserver v0.32.1 h1:hjkALhRUeCariC8DiVmb5jj0VjIc1N0DREP32+6UXZw= +k8s.io/apiextensions-apiserver v0.32.1/go.mod h1:sxWIGuGiYov7Io1fAS2X06NjMIk5CbRHc2StSmbaQto= +k8s.io/apimachinery v0.32.1 h1:683ENpaCBjma4CYqsmZyhEzrGz6cjn1MY/X2jB2hkZs= +k8s.io/apimachinery v0.32.1/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= +k8s.io/client-go v0.32.1 h1:otM0AxdhdBIaQh7l1Q0jQpmo7WOFIk5FFa4bg6YMdUU= +k8s.io/client-go v0.32.1/go.mod h1:aTTKZY7MdxUaJ/KiUs8D+GssR9zJZi77ZqtzcGXIiDg= +k8s.io/cloud-provider v0.32.1 h1:74rRhnfca3o4CsjjnIp/C3ARVuSmyNsxgWPtH0yc9Z0= +k8s.io/cloud-provider v0.32.1/go.mod h1:GECSanFT+EeZ/ToX3xlasjETzMUI+VFu92zHUDUsGHw= +k8s.io/component-base v0.32.1 h1:/5IfJ0dHIKBWysGV0yKTFfacZ5yNV1sulPh3ilJjRZk= +k8s.io/component-base v0.32.1/go.mod h1:j1iMMHi/sqAHeG5z+O9BFNCF698a1u0186zkjMZQ28w= +k8s.io/csi-translation-lib v0.32.1 h1:qqlB+eKiIdUM+GGZfJN/4FMNeuIPIELLxfWfv/LWUYk= +k8s.io/csi-translation-lib v0.32.1/go.mod h1:dc7zXqpUW4FykfAe6TqU32tYewsGhrjI63ZwJWQng3k= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f h1:GA7//TjRY9yWGy1poLzYYJJ4JRdzg3+O6e8I+e+8T5Y= k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f/go.mod h1:R/HEjbvWI0qdfb8viZUeVZm0X6IZnxAydC7YU42CMw4= k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -sigs.k8s.io/controller-runtime v0.19.4 h1:SUmheabttt0nx8uJtoII4oIP27BVVvAKFvdvGFwV/Qo= -sigs.k8s.io/controller-runtime v0.19.4/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4= +sigs.k8s.io/controller-runtime v0.20.0 h1:jjkMo29xEXH+02Md9qaVXfEIaMESSpy3TBWPrsfQkQs= +sigs.k8s.io/controller-runtime v0.20.0/go.mod h1:BrP3w158MwvB3ZbNpaAcIKkHQ7YGpYnzpoSTZ8E14WU= sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= sigs.k8s.io/structured-merge-diff/v4 v4.4.2 h1:MdmvkGuXi/8io6ixD5wud3vOLwc1rj0aNqRlpuvjmwA= From 5128b399d40f0cfd927597f9ef6657a65130a46a Mon Sep 17 00:00:00 2001 From: Jonathan Innis Date: Tue, 21 Jan 2025 12:26:36 -0800 Subject: [PATCH 03/17] ci: Drop stalebot from kubernetes-sigs/karpenter repo (#1924) --- .github/workflows/stale.yaml | 50 ------------------------------------ 1 file changed, 50 deletions(-) delete mode 100644 .github/workflows/stale.yaml diff --git a/.github/workflows/stale.yaml b/.github/workflows/stale.yaml deleted file mode 100644 index 25cbe05e89..0000000000 --- a/.github/workflows/stale.yaml +++ /dev/null @@ -1,50 +0,0 @@ -name: StaleBot -on: - workflow_dispatch: - schedule: - - cron: '0 12 * * *' -jobs: - StaleBot: - runs-on: ubuntu-latest - permissions: - issues: write # actions/stale@v8.0.0 - pull-requests: write # actions/stale@v8.0.0 - if: github.repository == 'kubernetes-sigs/karpenter' - name: Stale issue bot - steps: - # PR stale-out - - uses: actions/stale@28ca1036281a5e5922ead5184a1bbf96e5fc984e # v9.0.0 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - only-issue-labels: 'ignore' # Ignore this step for Issues - stale-pr-message: 'This PR has been inactive for 14 days. StaleBot will close this stale PR after 14 more days of inactivity.' - exempt-pr-labels: 'blocked,needs-review,needs-design' - stale-pr-label: 'lifecycle/stale' - close-pr-label: 'lifecycle/closed' - days-before-stale: 14 - days-before-close: 14 - operations-per-run: 300 - # Issue stale-out for "triage/needs-information" - - uses: actions/stale@28ca1036281a5e5922ead5184a1bbf96e5fc984e # v9.0.0 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - stale-issue-message: 'This issue has been inactive for 14 days. StaleBot will close this stale issue after 14 more days of inactivity.' - only-issue-labels: 'triage/needs-information' - stale-issue-label: 'lifecycle/stale' - close-issue-label: 'lifecycle/closed' - only-pr-labels: 'ignore' # Ignore this step for PRs - days-before-stale: 14 - days-before-close: 14 - operations-per-run: 300 - # Issue stale-out for "triage/solved" - - uses: actions/stale@28ca1036281a5e5922ead5184a1bbf96e5fc984e # v9.0.0 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - stale-issue-message: 'This issue has been inactive for 7 days and is marked as "triage/solved". StaleBot will close this stale issue after 7 more days of inactivity.' - only-issue-labels: 'triage/solved' - stale-issue-label: 'lifecycle/stale' - close-issue-label: 'lifecycle/closed' - only-pr-labels: 'ignore' # Ignore this step for PRs - days-before-stale: 7 - days-before-close: 7 - operations-per-run: 300 From af00eb443545048b99f1633bbecb8a12e9187c00 Mon Sep 17 00:00:00 2001 From: Jonathan Innis Date: Wed, 22 Jan 2025 14:40:37 -0800 Subject: [PATCH 04/17] BREAKING: Allow custom condition reasons to be returned from Create (#1925) --- pkg/cloudprovider/types.go | 4 +++- pkg/controllers/nodeclaim/lifecycle/launch.go | 2 +- pkg/controllers/nodeclaim/lifecycle/launch_test.go | 4 +++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/pkg/cloudprovider/types.go b/pkg/cloudprovider/types.go index d731c5310b..115c1afded 100644 --- a/pkg/cloudprovider/types.go +++ b/pkg/cloudprovider/types.go @@ -388,12 +388,14 @@ func IsNodeClassNotReadyError(err error) bool { // CreateError is an error type returned by CloudProviders when instance creation fails type CreateError struct { error + ConditionReason string ConditionMessage string } -func NewCreateError(err error, message string) *CreateError { +func NewCreateError(err error, reason, message string) *CreateError { return &CreateError{ error: err, + ConditionReason: reason, ConditionMessage: message, } } diff --git a/pkg/controllers/nodeclaim/lifecycle/launch.go b/pkg/controllers/nodeclaim/lifecycle/launch.go index 381b530562..8ec5801669 100644 --- a/pkg/controllers/nodeclaim/lifecycle/launch.go +++ b/pkg/controllers/nodeclaim/lifecycle/launch.go @@ -103,7 +103,7 @@ func (l *Launch) launchNodeClaim(ctx context.Context, nodeClaim *v1.NodeClaim) ( default: var createError *cloudprovider.CreateError if errors.As(err, &createError) { - nodeClaim.StatusConditions().SetUnknownWithReason(v1.ConditionTypeLaunched, "LaunchFailed", createError.ConditionMessage) + nodeClaim.StatusConditions().SetUnknownWithReason(v1.ConditionTypeLaunched, createError.ConditionReason, createError.ConditionMessage) } else { nodeClaim.StatusConditions().SetUnknownWithReason(v1.ConditionTypeLaunched, "LaunchFailed", truncateMessage(err.Error())) } diff --git a/pkg/controllers/nodeclaim/lifecycle/launch_test.go b/pkg/controllers/nodeclaim/lifecycle/launch_test.go index d25e04c6fe..19aa22c3fc 100644 --- a/pkg/controllers/nodeclaim/lifecycle/launch_test.go +++ b/pkg/controllers/nodeclaim/lifecycle/launch_test.go @@ -103,14 +103,16 @@ var _ = Describe("Launch", func() { ExpectNotFound(ctx, env.Client, nodeClaim) }) It("should set nodeClaim status condition from the condition message received if error returned is CreateError", func() { + conditionReason := "CustomReason" conditionMessage := "instance creation failed" - cloudProvider.NextCreateErr = cloudprovider.NewCreateError(fmt.Errorf("error launching instance"), conditionMessage) + cloudProvider.NextCreateErr = cloudprovider.NewCreateError(fmt.Errorf("error launching instance"), conditionReason, conditionMessage) nodeClaim := test.NodeClaim() ExpectApplied(ctx, env.Client, nodeClaim) _ = ExpectObjectReconcileFailed(ctx, env.Client, nodeClaimController, nodeClaim) nodeClaim = ExpectExists(ctx, env.Client, nodeClaim) condition := ExpectStatusConditionExists(nodeClaim, v1.ConditionTypeLaunched) Expect(condition.Status).To(Equal(metav1.ConditionUnknown)) + Expect(condition.Reason).To(Equal(conditionReason)) Expect(condition.Message).To(Equal(conditionMessage)) }) }) From c380935b0d9a14aca8b5925c6b276e88fb93f21f Mon Sep 17 00:00:00 2001 From: edibble21 <85638465+edibble21@users.noreply.github.com> Date: Fri, 24 Jan 2025 09:51:22 -0800 Subject: [PATCH 05/17] Chore: Version bump to 1.32 (#1927) --- .github/workflows/kind-e2e.yaml | 2 +- .github/workflows/presubmit.yaml | 4 ++-- hack/toolchain.sh | 2 +- kwok/apis/crds/karpenter.kwok.sh_kwoknodeclasses.yaml | 2 +- kwok/charts/crds/karpenter.sh_nodeclaims.yaml | 2 +- kwok/charts/crds/karpenter.sh_nodepools.yaml | 2 +- pkg/apis/crds/karpenter.sh_nodeclaims.yaml | 2 +- pkg/apis/crds/karpenter.sh_nodepools.yaml | 2 +- pkg/test/environment.go | 7 ++++--- .../v1alpha1/crds/karpenter.test.sh_testnodeclasses.yaml | 2 +- 10 files changed, 14 insertions(+), 13 deletions(-) diff --git a/.github/workflows/kind-e2e.yaml b/.github/workflows/kind-e2e.yaml index 224e6a8273..09a9f7afdf 100644 --- a/.github/workflows/kind-e2e.yaml +++ b/.github/workflows/kind-e2e.yaml @@ -9,7 +9,7 @@ jobs: strategy: fail-fast: false matrix: - k8sVersion: ["1.25.x", "1.26.x", "1.27.x", "1.28.x", "1.29.x", "1.30.x", "1.31.x"] + k8sVersion: ["1.25.x", "1.26.x", "1.27.x", "1.28.x", "1.29.x", "1.30.x", "1.31.x", "1.32.x"] steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Set up Python 3.10 diff --git a/.github/workflows/presubmit.yaml b/.github/workflows/presubmit.yaml index 78c8a50b65..2bdd328f8e 100644 --- a/.github/workflows/presubmit.yaml +++ b/.github/workflows/presubmit.yaml @@ -14,7 +14,7 @@ jobs: strategy: fail-fast: false matrix: - k8sVersion: ["1.25.x", "1.26.x", "1.27.x", "1.28.x", "1.29.x", "1.30.x", "1.31.x"] + k8sVersion: ["1.25.x", "1.26.x", "1.27.x", "1.28.x", "1.29.x", "1.30.x", "1.31.x", "1.32.x"] steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - uses: ./.github/actions/install-deps @@ -25,7 +25,7 @@ jobs: - run: K8S_VERSION=${{ matrix.k8sVersion }} make presubmit - name: Send coverage # should only send converage once https://docs.coveralls.io/parallel-builds - if: matrix.k8sVersion == '1.31.x' + if: matrix.k8sVersion == '1.32.x' env: COVERALLS_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: goveralls -coverprofile=coverage.out -service=github diff --git a/hack/toolchain.sh b/hack/toolchain.sh index d1bf76ed2d..b33f3b09d9 100755 --- a/hack/toolchain.sh +++ b/hack/toolchain.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail -K8S_VERSION="${K8S_VERSION:="1.31.x"}" +K8S_VERSION="${K8S_VERSION:="1.32.x"}" KUBEBUILDER_ASSETS="/usr/local/kubebuilder/bin" main() { diff --git a/kwok/apis/crds/karpenter.kwok.sh_kwoknodeclasses.yaml b/kwok/apis/crds/karpenter.kwok.sh_kwoknodeclasses.yaml index c93d3e75ca..e9a158a966 100644 --- a/kwok/apis/crds/karpenter.kwok.sh_kwoknodeclasses.yaml +++ b/kwok/apis/crds/karpenter.kwok.sh_kwoknodeclasses.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.16.5 + controller-gen.kubebuilder.io/version: v0.17.1 name: kwoknodeclasses.karpenter.kwok.sh spec: group: karpenter.kwok.sh diff --git a/kwok/charts/crds/karpenter.sh_nodeclaims.yaml b/kwok/charts/crds/karpenter.sh_nodeclaims.yaml index 1150d87ca5..92eb9dbe03 100644 --- a/kwok/charts/crds/karpenter.sh_nodeclaims.yaml +++ b/kwok/charts/crds/karpenter.sh_nodeclaims.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.16.5 + controller-gen.kubebuilder.io/version: v0.17.1 name: nodeclaims.karpenter.sh spec: group: karpenter.sh diff --git a/kwok/charts/crds/karpenter.sh_nodepools.yaml b/kwok/charts/crds/karpenter.sh_nodepools.yaml index 84ba89082f..bd608d655f 100644 --- a/kwok/charts/crds/karpenter.sh_nodepools.yaml +++ b/kwok/charts/crds/karpenter.sh_nodepools.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.16.5 + controller-gen.kubebuilder.io/version: v0.17.1 name: nodepools.karpenter.sh spec: group: karpenter.sh diff --git a/pkg/apis/crds/karpenter.sh_nodeclaims.yaml b/pkg/apis/crds/karpenter.sh_nodeclaims.yaml index 9465ed5b53..00018dc000 100644 --- a/pkg/apis/crds/karpenter.sh_nodeclaims.yaml +++ b/pkg/apis/crds/karpenter.sh_nodeclaims.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.16.5 + controller-gen.kubebuilder.io/version: v0.17.1 name: nodeclaims.karpenter.sh spec: group: karpenter.sh diff --git a/pkg/apis/crds/karpenter.sh_nodepools.yaml b/pkg/apis/crds/karpenter.sh_nodepools.yaml index 67d7d2d728..36ecac075d 100644 --- a/pkg/apis/crds/karpenter.sh_nodepools.yaml +++ b/pkg/apis/crds/karpenter.sh_nodepools.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.16.5 + controller-gen.kubebuilder.io/version: v0.17.1 name: nodepools.karpenter.sh spec: group: karpenter.sh diff --git a/pkg/test/environment.go b/pkg/test/environment.go index fc7a1dae7c..b04b66270b 100644 --- a/pkg/test/environment.go +++ b/pkg/test/environment.go @@ -114,16 +114,17 @@ func NewEnvironment(options ...option.Function[EnvironmentOptions]) *Environment opts := option.Resolve(options...) ctx, cancel := context.WithCancel(context.Background()) - version := version.MustParseSemantic(strings.Replace(env.WithDefaultString("K8S_VERSION", "1.31.x"), ".x", ".0", -1)) + version := version.MustParseSemantic(strings.Replace(env.WithDefaultString("K8S_VERSION", "1.32.x"), ".x", ".0", -1)) environment := envtest.Environment{Scheme: scheme.Scheme, CRDs: opts.crds} - if version.Minor() >= 21 { + if version.Minor() >= 21 && version.Minor() < 32 { // PodAffinityNamespaceSelector is used for label selectors in pod affinities. If the feature-gate is turned off, // the api-server just clears out the label selector so we never see it. If we turn it on, the label selectors // are passed to us and we handle them. This feature is alpha in apiextensionsv1.21, beta in apiextensionsv1.22 and will be GA in 1.24. See // https://github.com/kubernetes/enhancements/issues/2249 for more info. environment.ControlPlane.GetAPIServer().Configure().Set("feature-gates", "PodAffinityNamespaceSelector=true") } - if version.Minor() >= 24 { + //MinDomains got promoted to stable in 1.32 + if version.Minor() >= 24 && version.Minor() < 32 { // MinDomainsInPodTopologySpread enforces a minimum number of eligible node domains for pod scheduling // See https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/#spread-constraint-definition // Ref: https://github.com/aws/karpenter-core/pull/330 diff --git a/pkg/test/v1alpha1/crds/karpenter.test.sh_testnodeclasses.yaml b/pkg/test/v1alpha1/crds/karpenter.test.sh_testnodeclasses.yaml index d36c4e448c..a80a21f279 100644 --- a/pkg/test/v1alpha1/crds/karpenter.test.sh_testnodeclasses.yaml +++ b/pkg/test/v1alpha1/crds/karpenter.test.sh_testnodeclasses.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.16.5 + controller-gen.kubebuilder.io/version: v0.17.1 name: testnodeclasses.karpenter.test.sh spec: group: karpenter.test.sh From f9881a61538c3273f3516d1fc37c6e81edccbda6 Mon Sep 17 00:00:00 2001 From: Eng Zer Jun Date: Mon, 27 Jan 2025 14:21:23 +0800 Subject: [PATCH 06/17] chore: replace golang.org/x/exp with stdlib (#1931) Signed-off-by: Eng Zer Jun --- go.mod | 1 - go.sum | 2 -- pkg/utils/nodepool/suite_test.go | 4 ++-- pkg/utils/pretty/pretty.go | 6 +++--- 4 files changed, 5 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 259364fd36..59f8564355 100644 --- a/go.mod +++ b/go.mod @@ -66,7 +66,6 @@ require ( github.com/spf13/cobra v1.8.1 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/x448/float16 v0.8.4 // indirect - golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 golang.org/x/net v0.33.0 // indirect golang.org/x/oauth2 v0.23.0 // indirect golang.org/x/sys v0.28.0 // indirect diff --git a/go.sum b/go.sum index aed8014d0f..09eac80727 100644 --- a/go.sum +++ b/go.sum @@ -142,8 +142,6 @@ go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= diff --git a/pkg/utils/nodepool/suite_test.go b/pkg/utils/nodepool/suite_test.go index ead9695db5..890a865e3b 100644 --- a/pkg/utils/nodepool/suite_test.go +++ b/pkg/utils/nodepool/suite_test.go @@ -18,12 +18,12 @@ package nodepool_test import ( "context" + "math/rand/v2" "testing" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/samber/lo" - "golang.org/x/exp/rand" "sigs.k8s.io/karpenter/pkg/apis" v1 "sigs.k8s.io/karpenter/pkg/apis/v1" @@ -64,7 +64,7 @@ var _ = Describe("NodePoolUtils", func() { nps := lo.Shuffle(lo.Times(10, func(_ int) *v1.NodePool { return test.NodePool(v1.NodePool{ Spec: v1.NodePoolSpec{ - Weight: lo.ToPtr[int32](int32(rand.Intn(100) + 1)), //nolint:gosec + Weight: lo.ToPtr[int32](int32(rand.IntN(100) + 1)), //nolint:gosec }, }) })) diff --git a/pkg/utils/pretty/pretty.go b/pkg/utils/pretty/pretty.go index 009be00366..3694efdfa4 100644 --- a/pkg/utils/pretty/pretty.go +++ b/pkg/utils/pretty/pretty.go @@ -18,14 +18,14 @@ package pretty import ( "bytes" + "cmp" "encoding/json" "fmt" "regexp" + "slices" "strings" "unicode" - "golang.org/x/exp/constraints" - "golang.org/x/exp/slices" v1 "k8s.io/api/core/v1" ) @@ -55,7 +55,7 @@ func Slice[T any](s []T, maxItems int) string { // Map truncates a map after a certain number of max items to ensure that the // description in a log doesn't get too long -func Map[K constraints.Ordered, V any](values map[K]V, maxItems int) string { +func Map[K cmp.Ordered, V any](values map[K]V, maxItems int) string { var buf bytes.Buffer count := 0 var keys []K From bed80cc31ae51e4ce59b24c0498fa708373a9e5a Mon Sep 17 00:00:00 2001 From: Hansuk Hong Date: Mon, 27 Jan 2025 15:27:22 +0900 Subject: [PATCH 07/17] chore: move event reasons to the package and as constants (#1915) Signed-off-by: flavono123 --- .../disruption/consolidation_test.go | 12 ++--- pkg/controllers/disruption/events/events.go | 20 ++++---- pkg/controllers/node/health/events.go | 10 ++-- .../termination/terminator/events/events.go | 10 ++-- .../node/termination/terminator/suite_test.go | 13 ++--- .../nodeclaim/consistency/events.go | 2 +- pkg/controllers/nodeclaim/lifecycle/events.go | 4 +- .../provisioning/scheduling/events.go | 6 +-- pkg/events/reason.go | 48 +++++++++++++++++++ 9 files changed, 87 insertions(+), 38 deletions(-) create mode 100644 pkg/events/reason.go diff --git a/pkg/controllers/disruption/consolidation_test.go b/pkg/controllers/disruption/consolidation_test.go index d69554f1db..9506873470 100644 --- a/pkg/controllers/disruption/consolidation_test.go +++ b/pkg/controllers/disruption/consolidation_test.go @@ -112,7 +112,7 @@ var _ = Describe("Consolidation", func() { ExpectSingletonReconciled(ctx, disruptionController) wg.Wait() - Expect(recorder.Calls("Unconsolidatable")).To(Equal(0)) + Expect(recorder.Calls(events.Unconsolidatable)).To(Equal(0)) }) It("should fire an event for ConsolidationDisabled when the NodePool has consolidation set to WhenEmpty", func() { pod := test.Pod() @@ -123,7 +123,7 @@ var _ = Describe("Consolidation", func() { ExpectMakeNodesAndNodeClaimsInitializedAndStateUpdated(ctx, env.Client, nodeStateController, nodeClaimStateController, []*corev1.Node{node}, []*v1.NodeClaim{nodeClaim}) ExpectSingletonReconciled(ctx, disruptionController) - Expect(recorder.Calls("Unconsolidatable")).To(Equal(4)) + Expect(recorder.Calls(events.Unconsolidatable)).To(Equal(4)) }) It("should fire an event for ConsolidationDisabled when the NodePool has consolidateAfter set to 'Never'", func() { pod := test.Pod() @@ -135,7 +135,7 @@ var _ = Describe("Consolidation", func() { ExpectSingletonReconciled(ctx, disruptionController) // We get six calls here because we have Nodes and NodeClaims that fired for this event // and each of the consolidation mechanisms specifies that this event should be fired - Expect(recorder.Calls("Unconsolidatable")).To(Equal(6)) + Expect(recorder.Calls(events.Unconsolidatable)).To(Equal(6)) }) It("should fire an event when a candidate does not have a resolvable instance type", func() { pod := test.Pod() @@ -148,7 +148,7 @@ var _ = Describe("Consolidation", func() { ExpectMakeNodesAndNodeClaimsInitializedAndStateUpdated(ctx, env.Client, nodeStateController, nodeClaimStateController, []*corev1.Node{node}, []*v1.NodeClaim{nodeClaim}) ExpectSingletonReconciled(ctx, disruptionController) // We get four calls since we only care about this since we don't emit for empty node consolidation - Expect(recorder.Calls("Unconsolidatable")).To(Equal(4)) + Expect(recorder.Calls(events.Unconsolidatable)).To(Equal(4)) }) It("should fire an event when a candidate does not have the capacity type label", func() { pod := test.Pod() @@ -161,7 +161,7 @@ var _ = Describe("Consolidation", func() { ExpectMakeNodesAndNodeClaimsInitializedAndStateUpdated(ctx, env.Client, nodeStateController, nodeClaimStateController, []*corev1.Node{node}, []*v1.NodeClaim{nodeClaim}) ExpectSingletonReconciled(ctx, disruptionController) // We get four calls since we only care about this since we don't emit for empty node consolidation - Expect(recorder.Calls("Unconsolidatable")).To(Equal(4)) + Expect(recorder.Calls(events.Unconsolidatable)).To(Equal(4)) }) It("should fire an event when a candidate does not have the zone label", func() { pod := test.Pod() @@ -174,7 +174,7 @@ var _ = Describe("Consolidation", func() { ExpectMakeNodesAndNodeClaimsInitializedAndStateUpdated(ctx, env.Client, nodeStateController, nodeClaimStateController, []*corev1.Node{node}, []*v1.NodeClaim{nodeClaim}) ExpectSingletonReconciled(ctx, disruptionController) // We get four calls since we only care about this since we don't emit for empty node consolidation - Expect(recorder.Calls("Unconsolidatable")).To(Equal(4)) + Expect(recorder.Calls(events.Unconsolidatable)).To(Equal(4)) }) }) Context("Metrics", func() { diff --git a/pkg/controllers/disruption/events/events.go b/pkg/controllers/disruption/events/events.go index e40eec8b4a..263d4c326c 100644 --- a/pkg/controllers/disruption/events/events.go +++ b/pkg/controllers/disruption/events/events.go @@ -32,7 +32,7 @@ func Launching(nodeClaim *v1.NodeClaim, reason string) events.Event { return events.Event{ InvolvedObject: nodeClaim, Type: corev1.EventTypeNormal, - Reason: "DisruptionLaunching", + Reason: events.DisruptionLaunching, Message: fmt.Sprintf("Launching NodeClaim: %s", cases.Title(language.Und, cases.NoLower).String(reason)), DedupeValues: []string{string(nodeClaim.UID), reason}, } @@ -42,7 +42,7 @@ func WaitingOnReadiness(nodeClaim *v1.NodeClaim) events.Event { return events.Event{ InvolvedObject: nodeClaim, Type: corev1.EventTypeNormal, - Reason: "DisruptionWaitingReadiness", + Reason: events.DisruptionWaitingReadiness, Message: "Waiting on readiness to continue disruption", DedupeValues: []string{string(nodeClaim.UID)}, } @@ -53,14 +53,14 @@ func Terminating(node *corev1.Node, nodeClaim *v1.NodeClaim, reason string) []ev { InvolvedObject: node, Type: corev1.EventTypeNormal, - Reason: "DisruptionTerminating", + Reason: events.DisruptionTerminating, Message: fmt.Sprintf("Disrupting Node: %s", cases.Title(language.Und, cases.NoLower).String(reason)), DedupeValues: []string{string(node.UID), reason}, }, { InvolvedObject: nodeClaim, Type: corev1.EventTypeNormal, - Reason: "DisruptionTerminating", + Reason: events.DisruptionTerminating, Message: fmt.Sprintf("Disrupting NodeClaim: %s", cases.Title(language.Und, cases.NoLower).String(reason)), DedupeValues: []string{string(nodeClaim.UID), reason}, }, @@ -74,7 +74,7 @@ func Unconsolidatable(node *corev1.Node, nodeClaim *v1.NodeClaim, msg string) [] { InvolvedObject: node, Type: corev1.EventTypeNormal, - Reason: "Unconsolidatable", + Reason: events.Unconsolidatable, Message: msg, DedupeValues: []string{string(node.UID)}, DedupeTimeout: time.Minute * 15, @@ -82,7 +82,7 @@ func Unconsolidatable(node *corev1.Node, nodeClaim *v1.NodeClaim, msg string) [] { InvolvedObject: nodeClaim, Type: corev1.EventTypeNormal, - Reason: "Unconsolidatable", + Reason: events.Unconsolidatable, Message: msg, DedupeValues: []string{string(nodeClaim.UID)}, DedupeTimeout: time.Minute * 15, @@ -97,7 +97,7 @@ func Blocked(node *corev1.Node, nodeClaim *v1.NodeClaim, msg string) (evs []even evs = append(evs, events.Event{ InvolvedObject: node, Type: corev1.EventTypeNormal, - Reason: "DisruptionBlocked", + Reason: events.DisruptionBlocked, Message: msg, DedupeValues: []string{string(node.UID)}, }) @@ -106,7 +106,7 @@ func Blocked(node *corev1.Node, nodeClaim *v1.NodeClaim, msg string) (evs []even evs = append(evs, events.Event{ InvolvedObject: nodeClaim, Type: corev1.EventTypeNormal, - Reason: "DisruptionBlocked", + Reason: events.DisruptionBlocked, Message: msg, DedupeValues: []string{string(nodeClaim.UID)}, }) @@ -118,7 +118,7 @@ func NodePoolBlockedForDisruptionReason(nodePool *v1.NodePool, reason v1.Disrupt return events.Event{ InvolvedObject: nodePool, Type: corev1.EventTypeNormal, - Reason: "DisruptionBlocked", + Reason: events.DisruptionBlocked, Message: fmt.Sprintf("No allowed disruptions for disruption reason %s due to blocking budget", reason), DedupeValues: []string{string(nodePool.UID), string(reason)}, DedupeTimeout: 1 * time.Minute, @@ -129,7 +129,7 @@ func NodePoolBlocked(nodePool *v1.NodePool) events.Event { return events.Event{ InvolvedObject: nodePool, Type: corev1.EventTypeNormal, - Reason: "DisruptionBlocked", + Reason: events.DisruptionBlocked, Message: "No allowed disruptions due to blocking budget", DedupeValues: []string{string(nodePool.UID)}, // Set a small timeout as a NodePool's disruption budget can change every minute. diff --git a/pkg/controllers/node/health/events.go b/pkg/controllers/node/health/events.go index 8dfddaf7ca..2e45a40475 100644 --- a/pkg/controllers/node/health/events.go +++ b/pkg/controllers/node/health/events.go @@ -30,7 +30,7 @@ func NodeRepairBlocked(node *corev1.Node, nodeClaim *v1.NodeClaim, nodePool *v1. { InvolvedObject: node, Type: corev1.EventTypeWarning, - Reason: "NodeRepairBlocked", + Reason: events.NodeRepairBlocked, Message: reason, DedupeValues: []string{string(node.UID)}, DedupeTimeout: time.Minute * 15, @@ -38,7 +38,7 @@ func NodeRepairBlocked(node *corev1.Node, nodeClaim *v1.NodeClaim, nodePool *v1. { InvolvedObject: node, Type: corev1.EventTypeWarning, - Reason: "NodeRepairBlocked", + Reason: events.NodeRepairBlocked, Message: reason, DedupeValues: []string{string(nodeClaim.UID)}, DedupeTimeout: time.Minute * 15, @@ -46,7 +46,7 @@ func NodeRepairBlocked(node *corev1.Node, nodeClaim *v1.NodeClaim, nodePool *v1. { InvolvedObject: node, Type: corev1.EventTypeWarning, - Reason: "NodeRepairBlocked", + Reason: events.NodeRepairBlocked, Message: reason, DedupeValues: []string{string(nodePool.UID)}, DedupeTimeout: time.Minute * 15, @@ -59,7 +59,7 @@ func NodeRepairBlockedUnmanagedNodeClaim(node *corev1.Node, nodeClaim *v1.NodeCl { InvolvedObject: node, Type: corev1.EventTypeWarning, - Reason: "NodeRepairBlocked", + Reason: events.NodeRepairBlocked, Message: reason, DedupeValues: []string{string(node.UID)}, DedupeTimeout: time.Minute * 15, @@ -67,7 +67,7 @@ func NodeRepairBlockedUnmanagedNodeClaim(node *corev1.Node, nodeClaim *v1.NodeCl { InvolvedObject: node, Type: corev1.EventTypeWarning, - Reason: "NodeRepairBlocked", + Reason: events.NodeRepairBlocked, Message: reason, DedupeValues: []string{string(nodeClaim.UID)}, DedupeTimeout: time.Minute * 15, diff --git a/pkg/controllers/node/termination/terminator/events/events.go b/pkg/controllers/node/termination/terminator/events/events.go index 0e790978f3..8e816d06a8 100644 --- a/pkg/controllers/node/termination/terminator/events/events.go +++ b/pkg/controllers/node/termination/terminator/events/events.go @@ -30,7 +30,7 @@ func EvictPod(pod *corev1.Pod, message string) events.Event { return events.Event{ InvolvedObject: pod, Type: corev1.EventTypeNormal, - Reason: "Evicted", + Reason: events.Evicted, Message: "Evicted pod: " + message, DedupeValues: []string{pod.Name}, } @@ -40,7 +40,7 @@ func DisruptPodDelete(pod *corev1.Pod, gracePeriodSeconds *int64, nodeGracePerio return events.Event{ InvolvedObject: pod, Type: corev1.EventTypeNormal, - Reason: "Disrupted", + Reason: events.Disrupted, Message: fmt.Sprintf("Deleting the pod to accommodate the terminationTime %v of the node. The pod was granted %v seconds of grace-period of its %v terminationGracePeriodSeconds. This bypasses the PDB of the pod and the do-not-disrupt annotation.", *nodeGracePeriodTerminationTime, *gracePeriodSeconds, pod.Spec.TerminationGracePeriodSeconds), DedupeValues: []string{pod.Name}, } @@ -50,7 +50,7 @@ func NodeFailedToDrain(node *corev1.Node, err error) events.Event { return events.Event{ InvolvedObject: node, Type: corev1.EventTypeWarning, - Reason: "FailedDraining", + Reason: events.FailedDraining, Message: fmt.Sprintf("Failed to drain node, %s", err), DedupeValues: []string{node.Name}, } @@ -60,7 +60,7 @@ func NodeTerminationGracePeriodExpiring(node *corev1.Node, terminationTime strin return events.Event{ InvolvedObject: node, Type: corev1.EventTypeWarning, - Reason: "TerminationGracePeriodExpiring", + Reason: events.TerminationGracePeriodExpiring, Message: fmt.Sprintf("All pods will be deleted by %s", terminationTime), DedupeValues: []string{node.Name}, } @@ -70,7 +70,7 @@ func NodeClaimTerminationGracePeriodExpiring(nodeClaim *v1.NodeClaim, terminatio return events.Event{ InvolvedObject: nodeClaim, Type: corev1.EventTypeWarning, - Reason: "TerminationGracePeriodExpiring", + Reason: events.TerminationGracePeriodExpiring, Message: fmt.Sprintf("All pods will be deleted by %s", terminationTime), DedupeValues: []string{nodeClaim.Name}, } diff --git a/pkg/controllers/node/termination/terminator/suite_test.go b/pkg/controllers/node/termination/terminator/suite_test.go index 39bd3ce16b..86dd901a37 100644 --- a/pkg/controllers/node/termination/terminator/suite_test.go +++ b/pkg/controllers/node/termination/terminator/suite_test.go @@ -35,6 +35,7 @@ import ( "sigs.k8s.io/karpenter/pkg/apis" "sigs.k8s.io/karpenter/pkg/controllers/node/termination/terminator" + "sigs.k8s.io/karpenter/pkg/events" "sigs.k8s.io/karpenter/pkg/operator/options" "sigs.k8s.io/karpenter/pkg/test" . "sigs.k8s.io/karpenter/pkg/test/expectations" @@ -112,7 +113,7 @@ var _ = Describe("Eviction/Queue", func() { ExpectApplied(ctx, env.Client, pod) Expect(queue.Evict(ctx, terminator.NewQueueKey(pod, node.Spec.ProviderID))).To(BeTrue()) ExpectMetricCounterValue(terminator.NodesEvictionRequestsTotal, 1, map[string]string{terminator.CodeLabel: "200"}) - Expect(recorder.Calls("Evicted")).To(Equal(1)) + Expect(recorder.Calls(events.Evicted)).To(Equal(1)) }) It("should succeed with no event when there are PDBs that allow an eviction", func() { pdb = test.PodDisruptionBudget(test.PDBOptions{ @@ -121,12 +122,12 @@ var _ = Describe("Eviction/Queue", func() { }) ExpectApplied(ctx, env.Client, pod) Expect(queue.Evict(ctx, terminator.NewQueueKey(pod, node.Spec.ProviderID))).To(BeTrue()) - Expect(recorder.Calls("Evicted")).To(Equal(1)) + Expect(recorder.Calls(events.Evicted)).To(Equal(1)) }) It("should return a NodeDrainError event when a PDB is blocking", func() { ExpectApplied(ctx, env.Client, pdb, pod) Expect(queue.Evict(ctx, terminator.NewQueueKey(pod, node.Spec.ProviderID))).To(BeFalse()) - Expect(recorder.Calls("FailedDraining")).To(Equal(1)) + Expect(recorder.Calls(events.FailedDraining)).To(Equal(1)) }) It("should fail when two PDBs refer to the same pod", func() { pdb2 := test.PodDisruptionBudget(test.PDBOptions{ @@ -171,7 +172,7 @@ var _ = Describe("Eviction/Queue", func() { Expect(terminatorInstance.DeleteExpiringPods(ctx, []*corev1.Pod{pod}, nil)).To(Succeed()) ExpectExists(ctx, env.Client, pod) - Expect(recorder.Calls("Disrupted")).To(Equal(0)) + Expect(recorder.Calls(events.Disrupted)).To(Equal(0)) }) It("should not delete a pod with terminationGracePeriodSeconds still remaining before nodeTerminationTime", func() { pod.Spec.TerminationGracePeriodSeconds = lo.ToPtr[int64](60) @@ -180,7 +181,7 @@ var _ = Describe("Eviction/Queue", func() { nodeTerminationTime := time.Now().Add(time.Minute * 5) Expect(terminatorInstance.DeleteExpiringPods(ctx, []*corev1.Pod{pod}, &nodeTerminationTime)).To(Succeed()) ExpectExists(ctx, env.Client, pod) - Expect(recorder.Calls("Disrupted")).To(Equal(0)) + Expect(recorder.Calls(events.Disrupted)).To(Equal(0)) }) It("should delete a pod with less than terminationGracePeriodSeconds remaining before nodeTerminationTime", func() { pod.Spec.TerminationGracePeriodSeconds = lo.ToPtr[int64](120) @@ -189,7 +190,7 @@ var _ = Describe("Eviction/Queue", func() { nodeTerminationTime := time.Now().Add(time.Minute * 1) Expect(terminatorInstance.DeleteExpiringPods(ctx, []*corev1.Pod{pod}, &nodeTerminationTime)).To(Succeed()) ExpectNotFound(ctx, env.Client, pod) - Expect(recorder.Calls("Disrupted")).To(Equal(1)) + Expect(recorder.Calls(events.Disrupted)).To(Equal(1)) }) }) }) diff --git a/pkg/controllers/nodeclaim/consistency/events.go b/pkg/controllers/nodeclaim/consistency/events.go index 948d7bdfb1..d6ba447842 100644 --- a/pkg/controllers/nodeclaim/consistency/events.go +++ b/pkg/controllers/nodeclaim/consistency/events.go @@ -27,7 +27,7 @@ func FailedConsistencyCheckEvent(nodeClaim *v1.NodeClaim, message string) events return events.Event{ InvolvedObject: nodeClaim, Type: corev1.EventTypeWarning, - Reason: "FailedConsistencyCheck", + Reason: events.FailedConsistencyCheck, Message: message, DedupeValues: []string{string(nodeClaim.UID), message}, } diff --git a/pkg/controllers/nodeclaim/lifecycle/events.go b/pkg/controllers/nodeclaim/lifecycle/events.go index 2697b1cdc0..040cb03df9 100644 --- a/pkg/controllers/nodeclaim/lifecycle/events.go +++ b/pkg/controllers/nodeclaim/lifecycle/events.go @@ -29,7 +29,7 @@ func InsufficientCapacityErrorEvent(nodeClaim *v1.NodeClaim, err error) events.E return events.Event{ InvolvedObject: nodeClaim, Type: corev1.EventTypeWarning, - Reason: "InsufficientCapacityError", + Reason: events.InsufficientCapacityError, Message: fmt.Sprintf("NodeClaim %s event: %s", nodeClaim.Name, truncateMessage(err.Error())), DedupeValues: []string{string(nodeClaim.UID)}, } @@ -39,7 +39,7 @@ func NodeClassNotReadyEvent(nodeClaim *v1.NodeClaim, err error) events.Event { return events.Event{ InvolvedObject: nodeClaim, Type: corev1.EventTypeWarning, - Reason: "NodeClassNotReady", + Reason: events.NodeClassNotReady, Message: fmt.Sprintf("NodeClaim %s event: %s", nodeClaim.Name, truncateMessage(err.Error())), DedupeValues: []string{string(nodeClaim.UID)}, } diff --git a/pkg/controllers/provisioning/scheduling/events.go b/pkg/controllers/provisioning/scheduling/events.go index aa092475bc..0eff0a197b 100644 --- a/pkg/controllers/provisioning/scheduling/events.go +++ b/pkg/controllers/provisioning/scheduling/events.go @@ -42,7 +42,7 @@ func NominatePodEvent(pod *corev1.Pod, node *corev1.Node, nodeClaim *v1.NodeClai return events.Event{ InvolvedObject: pod, Type: corev1.EventTypeNormal, - Reason: "Nominated", + Reason: events.Nominated, Message: fmt.Sprintf("Pod should schedule on: %s", strings.Join(info, ", ")), DedupeValues: []string{string(pod.UID)}, RateLimiter: PodNominationRateLimiter, @@ -53,7 +53,7 @@ func NoCompatibleInstanceTypes(np *v1.NodePool) events.Event { return events.Event{ InvolvedObject: np, Type: corev1.EventTypeWarning, - Reason: "NoCompatibleInstanceTypes", + Reason: events.NoCompatibleInstanceTypes, Message: "NodePool requirements filtered out all compatible available instance types", DedupeValues: []string{string(np.UID)}, DedupeTimeout: 1 * time.Minute, @@ -64,7 +64,7 @@ func PodFailedToScheduleEvent(pod *corev1.Pod, err error) events.Event { return events.Event{ InvolvedObject: pod, Type: corev1.EventTypeWarning, - Reason: "FailedScheduling", + Reason: events.FailedScheduling, Message: fmt.Sprintf("Failed to schedule pod, %s", err), DedupeValues: []string{string(pod.UID)}, DedupeTimeout: 5 * time.Minute, diff --git a/pkg/events/reason.go b/pkg/events/reason.go new file mode 100644 index 0000000000..7d8ebc1fd0 --- /dev/null +++ b/pkg/events/reason.go @@ -0,0 +1,48 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package events + +// Reasons of events controllers emit +const ( + // disruption + DisruptionBlocked = "DisruptionBlocked" + DisruptionLaunching = "DisruptionLaunching" + DisruptionTerminating = "DisruptionTerminating" + DisruptionWaitingReadiness = "DisruptionWaitingReadiness" + Unconsolidatable = "Unconsolidatable" + + // provisioning/scheduling + FailedScheduling = "FailedScheduling" + NoCompatibleInstanceTypes = "NoCompatibleInstanceTypes" + Nominated = "Nominated" + + // node/health + NodeRepairBlocked = "NodeRepairBlocked" + + // node/termination/terminator + Disrupted = "Disrupted" + Evicted = "Evicted" + FailedDraining = "FailedDraining" + TerminationGracePeriodExpiring = "TerminationGracePeriodExpiring" + + // nodeclaim/consistency + FailedConsistencyCheck = "FailedConsistencyCheck" + + // nodeclaim/lifecycle + InsufficientCapacityError = "InsufficientCapacityError" + NodeClassNotReady = "NodeClassNotReady" +) From 28c0d196d23af53921763afac5c573a94f556192 Mon Sep 17 00:00:00 2001 From: Jonathan Innis Date: Mon, 27 Jan 2025 08:49:23 -0800 Subject: [PATCH 08/17] chore: Use non-deprecated TypedRateLimitingQueue (#1933) --- .../disruption/orchestration/queue.go | 21 +++++++------- .../disruption/orchestration/suite_test.go | 2 +- pkg/controllers/disruption/suite_test.go | 4 +-- pkg/test/ratelimitinginterface.go | 29 +++++++++---------- 4 files changed, 26 insertions(+), 30 deletions(-) diff --git a/pkg/controllers/disruption/orchestration/queue.go b/pkg/controllers/disruption/orchestration/queue.go index afe1cab4cd..17e85c344a 100644 --- a/pkg/controllers/disruption/orchestration/queue.go +++ b/pkg/controllers/disruption/orchestration/queue.go @@ -106,7 +106,7 @@ func IsUnrecoverableError(err error) bool { } type Queue struct { - workqueue.RateLimitingInterface + workqueue.TypedRateLimitingInterface[*Command] mu sync.RWMutex providerIDToCommand map[string]*Command // providerID -> command, maps a candidate to its command @@ -125,9 +125,9 @@ func NewQueue(kubeClient client.Client, recorder events.Recorder, cluster *state queue := &Queue{ // nolint:staticcheck // We need to implement a deprecated interface since Command currently doesn't implement "comparable" - RateLimitingInterface: workqueue.NewRateLimitingQueueWithConfig( - workqueue.NewItemExponentialFailureRateLimiter(queueBaseDelay, queueMaxDelay), - workqueue.RateLimitingQueueConfig{ + TypedRateLimitingInterface: workqueue.NewTypedRateLimitingQueueWithConfig( + workqueue.NewTypedItemExponentialFailureRateLimiter[*Command](queueBaseDelay, queueMaxDelay), + workqueue.TypedRateLimitingQueueConfig[*Command]{ Name: "disruption.workqueue", }), providerIDToCommand: map[string]*Command{}, @@ -171,11 +171,10 @@ func (q *Queue) Reconcile(ctx context.Context) (reconcile.Result, error) { } // Get command from queue. This waits until queue is non-empty. - item, shutdown := q.RateLimitingInterface.Get() + cmd, shutdown := q.TypedRateLimitingInterface.Get() if shutdown { panic("unexpected failure, disruption queue has shut down") } - cmd := item.(*Command) ctx = log.IntoContext(ctx, log.FromContext(ctx).WithValues("command-id", string(cmd.id))) if err := q.waitOrTerminate(ctx, cmd); err != nil { @@ -184,8 +183,8 @@ func (q *Queue) Reconcile(ctx context.Context) (reconcile.Result, error) { // store the error that is causing us to fail, so we can bubble it up later if this times out. cmd.lastError = err // mark this item as done processing. This is necessary so that the RLI is able to add the item back in. - q.RateLimitingInterface.Done(cmd) - q.RateLimitingInterface.AddRateLimited(cmd) + q.TypedRateLimitingInterface.Done(cmd) + q.TypedRateLimitingInterface.AddRateLimited(cmd) return reconcile.Result{RequeueAfter: singleton.RequeueImmediately}, nil } // If the command failed, bail on the action. @@ -298,7 +297,7 @@ func (q *Queue) Add(cmd *Command) error { q.providerIDToCommand[candidate.ProviderID()] = cmd } q.mu.Unlock() - q.RateLimitingInterface.Add(cmd) + q.TypedRateLimitingInterface.Add(cmd) return nil } @@ -318,8 +317,8 @@ func (q *Queue) HasAny(ids ...string) bool { // Remove fully clears the queue of all references of a hash/command func (q *Queue) Remove(cmd *Command) { // mark this item as done processing. This is necessary so that the RLI is able to add the item back in. - q.RateLimitingInterface.Done(cmd) - q.RateLimitingInterface.Forget(cmd) + q.TypedRateLimitingInterface.Done(cmd) + q.TypedRateLimitingInterface.Forget(cmd) q.cluster.UnmarkForDeletion(lo.Map(cmd.candidates, func(s *state.StateNode, _ int) string { return s.ProviderID() })...) // Remove all candidates linked to the command q.mu.Lock() diff --git a/pkg/controllers/disruption/orchestration/suite_test.go b/pkg/controllers/disruption/orchestration/suite_test.go index 78a3c1260c..3a5292e78e 100644 --- a/pkg/controllers/disruption/orchestration/suite_test.go +++ b/pkg/controllers/disruption/orchestration/suite_test.go @@ -355,6 +355,6 @@ func NewTestingQueue(kubeClient client.Client, recorder events.Recorder, cluster q := orchestration.NewQueue(kubeClient, recorder, cluster, clock, provisioner) // nolint:staticcheck // We need to implement a deprecated interface since Command currently doesn't implement "comparable" - q.RateLimitingInterface = test.NewRateLimitingInterface(workqueue.QueueConfig{Name: "disruption.workqueue"}) + q.TypedRateLimitingInterface = test.NewTypedRateLimitingInterface[*orchestration.Command](workqueue.TypedQueueConfig[*orchestration.Command]{Name: "disruption.workqueue"}) return q } diff --git a/pkg/controllers/disruption/suite_test.go b/pkg/controllers/disruption/suite_test.go index 54f2eb7a4e..42cd1afd38 100644 --- a/pkg/controllers/disruption/suite_test.go +++ b/pkg/controllers/disruption/suite_test.go @@ -2151,8 +2151,6 @@ func NewTestingQueue(kubeClient client.Client, recorder events.Recorder, cluster provisioner *provisioning.Provisioner) *orchestration.Queue { q := orchestration.NewQueue(kubeClient, recorder, cluster, clock, provisioner) - // nolint:staticcheck - // We need to implement a deprecated interface since Command currently doesn't implement "comparable" - q.RateLimitingInterface = test.NewRateLimitingInterface(workqueue.QueueConfig{Name: "disruption.workqueue"}) + q.TypedRateLimitingInterface = test.NewTypedRateLimitingInterface[*orchestration.Command](workqueue.TypedQueueConfig[*orchestration.Command]{Name: "disruption.workqueue"}) return q } diff --git a/pkg/test/ratelimitinginterface.go b/pkg/test/ratelimitinginterface.go index b894ce858b..c7ec656426 100644 --- a/pkg/test/ratelimitinginterface.go +++ b/pkg/test/ratelimitinginterface.go @@ -24,38 +24,37 @@ import ( "k8s.io/client-go/util/workqueue" ) -// RateLimitingInterface is copied from https://github.com/kubernetes-sigs/controller-runtime/blob/e6c3d139d2b6c286b1dbba6b6a95919159cfe655/pkg/controller/controllertest/testing.go#L48 -// but is an interface{} implementation to use while the orchestration queue can't use the TypedQueue +// TypedRateLimitingInterface is copied from https://github.com/kubernetes-sigs/controller-runtime/blob/e6c3d139d2b6c286b1dbba6b6a95919159cfe655/pkg/controller/controllertest/testing.go#L48 -// RateLimitingInterface implements a RateLimiting queue as a non-ratelimited queue for testing. +// TypedRateLimitingInterface implements a TypedRateLimiting queue as a non-ratelimited queue for testing. // This helps testing by having functions that use a RateLimiting queue synchronously add items to the queue. -type RateLimitingInterface struct { - workqueue.Interface +type TypedRateLimitingInterface[T comparable] struct { + *workqueue.Typed[T] AddedRateLimitedLock sync.Mutex AddedRatelimited []any } -func NewRateLimitingInterface(queueConfig workqueue.QueueConfig) *RateLimitingInterface { - return &RateLimitingInterface{Interface: workqueue.NewWithConfig(queueConfig)} +func NewTypedRateLimitingInterface[T comparable](queueConfig workqueue.TypedQueueConfig[T]) *TypedRateLimitingInterface[T] { + return &TypedRateLimitingInterface[T]{Typed: workqueue.NewTypedWithConfig[T](queueConfig)} } -// AddAfter implements RateLimitingInterface. -func (q *RateLimitingInterface) AddAfter(item interface{}, duration time.Duration) { +// AddAfter implements TypedRateLimitingInterface. +func (q *TypedRateLimitingInterface[T]) AddAfter(item T, duration time.Duration) { q.Add(item) } -// AddRateLimited implements RateLimitingInterface. -func (q *RateLimitingInterface) AddRateLimited(item interface{}) { +// AddRateLimited implements TypedRateLimitingInterface. +func (q *TypedRateLimitingInterface[T]) AddRateLimited(item T) { q.AddedRateLimitedLock.Lock() q.AddedRatelimited = append(q.AddedRatelimited, item) q.AddedRateLimitedLock.Unlock() q.Add(item) } -// Forget implements RateLimitingInterface. -func (q *RateLimitingInterface) Forget(item interface{}) {} +// Forget implements TypedRateLimitingInterface. +func (q *TypedRateLimitingInterface[T]) Forget(item T) {} -// NumRequeues implements RateLimitingInterface. -func (q *RateLimitingInterface) NumRequeues(item interface{}) int { +// NumRequeues implements TypedRateLimitingInterface. +func (q *TypedRateLimitingInterface[T]) NumRequeues(item T) int { return 0 } From 07e3d6e0dc96e168b9d144d309499c73e2ea4766 Mon Sep 17 00:00:00 2001 From: Jonathan Innis Date: Mon, 27 Jan 2025 09:19:23 -0800 Subject: [PATCH 09/17] fix: Fix scheduling benchmarking to not bail due to limits or same UUID (#1930) --- .../scheduling/scheduling_benchmark_test.go | 54 +++++++++---------- 1 file changed, 24 insertions(+), 30 deletions(-) diff --git a/pkg/controllers/provisioning/scheduling/scheduling_benchmark_test.go b/pkg/controllers/provisioning/scheduling/scheduling_benchmark_test.go index 2844b95362..814d06c196 100644 --- a/pkg/controllers/provisioning/scheduling/scheduling_benchmark_test.go +++ b/pkg/controllers/provisioning/scheduling/scheduling_benchmark_test.go @@ -20,7 +20,6 @@ package scheduling_test import ( "context" - "flag" "fmt" "math" "math/rand" @@ -35,6 +34,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/client-go/tools/record" "k8s.io/utils/clock" fakecr "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -59,9 +59,6 @@ var r = rand.New(rand.NewSource(42)) // To run the benchmarks use: // `go test -tags=test_performance -run=XXX -bench=.` // -// To run the benchmarks with minValues included in NodePool requirements, use: -// `go test -tags=test_performance -run=XXX -bench=. -minValues=true` -// // to get something statistically significant for comparison we need to run them several times and then // compare the results between the old performance and the new performance. // ```sh @@ -94,12 +91,6 @@ func BenchmarkScheduling5000(b *testing.B) { benchmarkScheduler(b, 400, 5000) } -var includeMinValues bool - -func init() { - flag.BoolVar(&includeMinValues, "minValues", false, "include minValues in NodePool requirement") -} - // TestSchedulingProfile is used to gather profiling metrics, benchmarking is primarily done with standard // Go benchmark functions // go test -tags=test_performance -run=SchedulingProfile @@ -133,32 +124,23 @@ func TestSchedulingProfile(t *testing.T) { totalNodes += int(nodeCount) } } - fmt.Println("scheduled", totalPods, "against", totalNodes, "nodes in total in", totalTime, "with minValues included", includeMinValues, float64(totalPods)/totalTime.Seconds(), "pods/sec") + fmt.Println("scheduled", totalPods, "against", totalNodes, "nodes in total in", totalTime, float64(totalPods)/totalTime.Seconds(), "pods/sec") tw.Flush() } func benchmarkScheduler(b *testing.B, instanceCount, podCount int) { // disable logging ctx = ctrl.IntoContext(context.Background(), operatorlogging.NopLogger) - nodePoolWithMinValues := test.NodePool(v1.NodePool{ + nodePool := test.NodePool(v1.NodePool{ Spec: v1.NodePoolSpec{ - Template: v1.NodeClaimTemplate{ - Spec: v1.NodeClaimTemplateSpec{ - Requirements: []v1.NodeSelectorRequirementWithMinValues{ - { - NodeSelectorRequirement: corev1.NodeSelectorRequirement{ - Key: corev1.LabelInstanceTypeStable, - Operator: corev1.NodeSelectorOpExists, - }, - MinValues: lo.ToPtr(50), // Adding highest possible minValues and safest way to add it would be to instanceType requirement. - }, - }, - }, + Limits: v1.Limits{ + corev1.ResourceCPU: resource.MustParse("10000000"), + corev1.ResourceMemory: resource.MustParse("10000000Gi"), }, }, }) - nodePoolWithoutMinValues := test.NodePool() - nodePool := lo.Ternary(includeMinValues, nodePoolWithMinValues, nodePoolWithoutMinValues) + + // Apply limits to both of the NodePools instanceTypes := fake.InstanceTypes(instanceCount) cloudProvider = fake.NewCloudProvider() cloudProvider.InstanceTypes = instanceTypes @@ -255,7 +237,10 @@ func makePodAntiAffinityPods(count int, key string) []*corev1.Pod { for i := 0; i < count; i++ { pods = append(pods, test.Pod( test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{Labels: labels}, + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + UID: uuid.NewUUID(), + }, PodAntiRequirements: []corev1.PodAffinityTerm{ { LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, @@ -276,7 +261,10 @@ func makePodAffinityPods(count int, key string) []*corev1.Pod { for i := 0; i < count; i++ { pods = append(pods, test.Pod( test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{Labels: randomAffinityLabels()}, + ObjectMeta: metav1.ObjectMeta{ + Labels: randomAffinityLabels(), + UID: uuid.NewUUID(), + }, PodRequirements: []corev1.PodAffinityTerm{ { LabelSelector: &metav1.LabelSelector{MatchLabels: randomAffinityLabels()}, @@ -298,7 +286,10 @@ func makeTopologySpreadPods(count int, key string) []*corev1.Pod { for i := 0; i < count; i++ { pods = append(pods, test.Pod( test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{Labels: randomLabels()}, + ObjectMeta: metav1.ObjectMeta{ + Labels: randomLabels(), + UID: uuid.NewUUID(), + }, TopologySpreadConstraints: []corev1.TopologySpreadConstraint{ { MaxSkew: 1, @@ -324,7 +315,10 @@ func makeGenericPods(count int) []*corev1.Pod { for i := 0; i < count; i++ { pods = append(pods, test.Pod( test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{Labels: randomLabels()}, + ObjectMeta: metav1.ObjectMeta{ + Labels: randomLabels(), + UID: uuid.NewUUID(), + }, ResourceRequirements: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceCPU: randomCPU(), From 554f2a96236cc401c38699658082aa3c83aa143b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 28 Jan 2025 08:17:25 -0800 Subject: [PATCH 10/17] chore(deps): bump sigs.k8s.io/controller-runtime from 0.20.0 to 0.20.1 in the k8s-go-deps group (#1936) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 59f8564355..1bd57ea2b1 100644 --- a/go.mod +++ b/go.mod @@ -30,7 +30,7 @@ require ( k8s.io/csi-translation-lib v0.32.1 k8s.io/klog/v2 v2.130.1 k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 - sigs.k8s.io/controller-runtime v0.20.0 + sigs.k8s.io/controller-runtime v0.20.1 ) require ( diff --git a/go.sum b/go.sum index 09eac80727..2716929b88 100644 --- a/go.sum +++ b/go.sum @@ -214,8 +214,8 @@ k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f h1:GA7//TjRY9yWGy1poLzYYJ k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f/go.mod h1:R/HEjbvWI0qdfb8viZUeVZm0X6IZnxAydC7YU42CMw4= k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -sigs.k8s.io/controller-runtime v0.20.0 h1:jjkMo29xEXH+02Md9qaVXfEIaMESSpy3TBWPrsfQkQs= -sigs.k8s.io/controller-runtime v0.20.0/go.mod h1:BrP3w158MwvB3ZbNpaAcIKkHQ7YGpYnzpoSTZ8E14WU= +sigs.k8s.io/controller-runtime v0.20.1 h1:JbGMAG/X94NeM3xvjenVUaBjy6Ui4Ogd/J5ZtjZnHaE= +sigs.k8s.io/controller-runtime v0.20.1/go.mod h1:BrP3w158MwvB3ZbNpaAcIKkHQ7YGpYnzpoSTZ8E14WU= sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= sigs.k8s.io/structured-merge-diff/v4 v4.4.2 h1:MdmvkGuXi/8io6ixD5wud3vOLwc1rj0aNqRlpuvjmwA= From 96d4bcef19a3133abf0e419952eaf78b7fee097c Mon Sep 17 00:00:00 2001 From: Jonathan Innis Date: Tue, 28 Jan 2025 09:17:24 -0800 Subject: [PATCH 11/17] chore: Bump go minor version to `1.23.5` (#1940) --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 1bd57ea2b1..c39b0103d5 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module sigs.k8s.io/karpenter -go 1.23.2 +go 1.23.5 require ( github.com/Pallinder/go-randomdata v1.2.0 From e9fea225adc564c34b9111e6b51dd1ed5be7777d Mon Sep 17 00:00:00 2001 From: Jonathan Innis Date: Tue, 28 Jan 2025 11:09:23 -0800 Subject: [PATCH 12/17] chore: Use self-affinity to ensure all pods schedule (#1938) --- .../scheduling/scheduling_benchmark_test.go | 47 +++++++++++++++---- 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/pkg/controllers/provisioning/scheduling/scheduling_benchmark_test.go b/pkg/controllers/provisioning/scheduling/scheduling_benchmark_test.go index 814d06c196..b08f105646 100644 --- a/pkg/controllers/provisioning/scheduling/scheduling_benchmark_test.go +++ b/pkg/controllers/provisioning/scheduling/scheduling_benchmark_test.go @@ -90,6 +90,12 @@ func BenchmarkScheduling2000(b *testing.B) { func BenchmarkScheduling5000(b *testing.B) { benchmarkScheduler(b, 400, 5000) } +func BenchmarkScheduling10000(b *testing.B) { + benchmarkScheduler(b, 400, 10000) +} +func BenchmarkScheduling20000(b *testing.B) { + benchmarkScheduler(b, 400, 20000) +} // TestSchedulingProfile is used to gather profiling metrics, benchmarking is primarily done with standard // Go benchmark functions @@ -114,7 +120,7 @@ func TestSchedulingProfile(t *testing.T) { totalNodes := 0 var totalTime time.Duration for _, instanceCount := range []int{400} { - for _, podCount := range []int{10, 100, 500, 1000, 1500, 2000, 5000} { + for _, podCount := range []int{1, 50, 100, 500, 1000, 1500, 2000, 5000, 10000, 20000} { start := time.Now() res := testing.Benchmark(func(b *testing.B) { benchmarkScheduler(b, instanceCount, podCount) }) totalTime += time.Since(start) / time.Duration(res.N) @@ -128,6 +134,7 @@ func TestSchedulingProfile(t *testing.T) { tw.Flush() } +// nolint:gocyclo func benchmarkScheduler(b *testing.B, instanceCount, podCount int) { // disable logging ctx = ctrl.IntoContext(context.Background(), operatorlogging.NopLogger) @@ -149,8 +156,8 @@ func benchmarkScheduler(b *testing.B, instanceCount, podCount int) { pods := makeDiversePods(podCount) clock := &clock.RealClock{} cluster = state.NewCluster(clock, client, cloudProvider) - domains := map[string]sets.Set[string]{} - topology, err := scheduling.NewTopology(ctx, client, cluster, domains, pods) + + topology, err := scheduling.NewTopology(ctx, client, cluster, getDomains(instanceTypes), pods) if err != nil { b.Fatalf("creating topology, %s", err) } @@ -167,8 +174,10 @@ func benchmarkScheduler(b *testing.B, instanceCount, podCount int) { nodesInRound1 := 0 for i := 0; i < b.N; i++ { results := scheduler.Solve(ctx, pods) + if len(results.PodErrors) > 0 { + b.Fatalf("expected all pods to schedule, got %d pods that didn't", len(results.PodErrors)) + } if i == 0 { - minPods := math.MaxInt64 maxPods := 0 var podCounts []int @@ -212,13 +221,30 @@ func benchmarkScheduler(b *testing.B, instanceCount, podCount int) { } } +func getDomains(instanceTypes []*cloudprovider.InstanceType) map[string]sets.Set[string] { + domains := map[string]sets.Set[string]{} + for _, it := range instanceTypes { + for key, requirement := range it.Requirements { + // This code used to execute a Union between domains[key] and requirement.Values(). + // The downside of this is that Union is immutable and takes a copy of the set it is executed upon. + // This resulted in a lot of memory pressure on the heap and poor performance + // https://github.com/aws/karpenter/issues/3565 + if domains[key] == nil { + domains[key] = sets.New(requirement.Values()...) + } else { + domains[key].Insert(requirement.Values()...) + } + } + } + return domains +} + func makeDiversePods(count int) []*corev1.Pod { var pods []*corev1.Pod - numTypes := 6 + numTypes := 5 pods = append(pods, makeGenericPods(count/numTypes)...) pods = append(pods, makeTopologySpreadPods(count/numTypes, corev1.LabelTopologyZone)...) pods = append(pods, makeTopologySpreadPods(count/numTypes, corev1.LabelHostname)...) - pods = append(pods, makePodAffinityPods(count/numTypes, corev1.LabelHostname)...) pods = append(pods, makePodAffinityPods(count/numTypes, corev1.LabelTopologyZone)...) pods = append(pods, makePodAntiAffinityPods(count/numTypes, corev1.LabelHostname)...) @@ -259,15 +285,20 @@ func makePodAntiAffinityPods(count int, key string) []*corev1.Pod { func makePodAffinityPods(count int, key string) []*corev1.Pod { var pods []*corev1.Pod for i := 0; i < count; i++ { + // We use self-affinity here because using affinity that relies on other pod + // domains doens't guarantee that all pods can schedule. In the case where you are not + // using self-affinity and the domain doesn't exist, scheduling will fail for all pods with + // affinities against this domain + labels := randomAffinityLabels() pods = append(pods, test.Pod( test.PodOptions{ ObjectMeta: metav1.ObjectMeta{ - Labels: randomAffinityLabels(), + Labels: labels, UID: uuid.NewUUID(), }, PodRequirements: []corev1.PodAffinityTerm{ { - LabelSelector: &metav1.LabelSelector{MatchLabels: randomAffinityLabels()}, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, TopologyKey: key, }, }, From bf825682f0e77eb5bf79460f6330a0ea8b53f565 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 28 Jan 2025 11:13:23 -0800 Subject: [PATCH 13/17] chore(deps): bump actions/setup-go from 5.2.0 to 5.3.0 in /.github/actions/install-deps in the action-deps group (#1935) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/actions/install-deps/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/install-deps/action.yaml b/.github/actions/install-deps/action.yaml index 03078017b5..aba23bb8f9 100644 --- a/.github/actions/install-deps/action.yaml +++ b/.github/actions/install-deps/action.yaml @@ -7,7 +7,7 @@ inputs: runs: using: "composite" steps: - - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0 + - uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0 id: setup-go with: go-version-file: go.mod From 2a09110a1cb68e1101c75c00d6f8407d3ed859a3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 28 Jan 2025 11:45:23 -0800 Subject: [PATCH 14/17] chore(deps): bump the go-deps group across 1 directory with 2 updates (#1941) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Jonathan Innis --- go.mod | 4 ++-- go.sum | 12 ++++++------ pkg/utils/nodepool/suite_test.go | 11 +++++++---- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index c39b0103d5..9844814aef 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/Pallinder/go-randomdata v1.2.0 github.com/avast/retry-go v3.0.0+incompatible github.com/awslabs/operatorpkg v0.0.0-20241205163410-0fff9f28d115 - github.com/docker/docker v27.5.0+incompatible + github.com/docker/docker v27.5.1+incompatible github.com/go-logr/logr v1.4.2 github.com/imdario/mergo v0.3.16 github.com/klauspost/compress v1.17.9 // indirect @@ -16,7 +16,7 @@ require ( github.com/patrickmn/go-cache v2.1.0+incompatible github.com/prometheus/client_golang v1.20.5 github.com/prometheus/client_model v0.6.1 - github.com/samber/lo v1.47.0 + github.com/samber/lo v1.49.1 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.0 golang.org/x/text v0.21.0 diff --git a/go.sum b/go.sum index 2716929b88..79605b9271 100644 --- a/go.sum +++ b/go.sum @@ -14,8 +14,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/docker/docker v27.5.0+incompatible h1:um++2NcQtGRTz5eEgO6aJimo6/JxrTXC941hd05JO6U= -github.com/docker/docker v27.5.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v27.5.1+incompatible h1:4PYU5dnBYqRQi0294d1FBECqT9ECWeQAIfE8q4YnPY8= +github.com/docker/docker v27.5.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= @@ -114,8 +114,8 @@ github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzG github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/samber/lo v1.47.0 h1:z7RynLwP5nbyRscyvcD043DWYoOcYRv3mV8lBeqOCLc= -github.com/samber/lo v1.47.0/go.mod h1:RmDH9Ct32Qy3gduHQuKJ3gW1fMHAnE/fAzQuf6He5cU= +github.com/samber/lo v1.49.1 h1:4BIFyVfuQSEpluc7Fua+j1NolZHiEHEpaSEKdsH0tew= +github.com/samber/lo v1.49.1/go.mod h1:dO6KHFzUKXgP8LDhU0oI8d2hekjXnGOu0DB8Jecxd6o= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -127,8 +127,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/pkg/utils/nodepool/suite_test.go b/pkg/utils/nodepool/suite_test.go index 890a865e3b..18a2932a5d 100644 --- a/pkg/utils/nodepool/suite_test.go +++ b/pkg/utils/nodepool/suite_test.go @@ -24,6 +24,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/samber/lo" + "github.com/samber/lo/mutable" "sigs.k8s.io/karpenter/pkg/apis" v1 "sigs.k8s.io/karpenter/pkg/apis/v1" @@ -61,13 +62,14 @@ var _ = Describe("NodePoolUtils", func() { Context("OrderByWeight", func() { It("should order the NodePools by weight", func() { // Generate 10 NodePools that have random weights, some might have the same weights - nps := lo.Shuffle(lo.Times(10, func(_ int) *v1.NodePool { + nps := lo.Times(10, func(_ int) *v1.NodePool { return test.NodePool(v1.NodePool{ Spec: v1.NodePoolSpec{ Weight: lo.ToPtr[int32](int32(rand.IntN(100) + 1)), //nolint:gosec }, }) - })) + }) + mutable.Shuffle(nps) nodepoolutils.OrderByWeight(nps) lastWeight := 101 // This is above the allowed weight values @@ -78,13 +80,14 @@ var _ = Describe("NodePoolUtils", func() { }) It("should order the NodePools by name when the weights are the same", func() { // Generate 10 NodePools with the same weight - nps := lo.Shuffle(lo.Times(10, func(_ int) *v1.NodePool { + nps := lo.Times(10, func(_ int) *v1.NodePool { return test.NodePool(v1.NodePool{ Spec: v1.NodePoolSpec{ Weight: lo.ToPtr[int32](10), }, }) - })) + }) + mutable.Shuffle(nps) nodepoolutils.OrderByWeight(nps) lastName := "zzzzzzzzzzzzzzzzzzzzzzzz" // large string value From 1e21a7d534a5c960960c36f7eebbdfa5d3d8a55e Mon Sep 17 00:00:00 2001 From: Jonathan Innis Date: Fri, 31 Jan 2025 12:40:56 -0800 Subject: [PATCH 15/17] fix: Fix heap profile benchmarking (#1946) --- .../provisioning/scheduling/scheduling_benchmark_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controllers/provisioning/scheduling/scheduling_benchmark_test.go b/pkg/controllers/provisioning/scheduling/scheduling_benchmark_test.go index b08f105646..ba45005a5c 100644 --- a/pkg/controllers/provisioning/scheduling/scheduling_benchmark_test.go +++ b/pkg/controllers/provisioning/scheduling/scheduling_benchmark_test.go @@ -114,7 +114,7 @@ func TestSchedulingProfile(t *testing.T) { if err != nil { t.Fatalf("error creating heap profile: %s", err) } - defer lo.Must0(pprof.WriteHeapProfile(heapf)) + defer func() { lo.Must0(pprof.WriteHeapProfile(heapf)) }() totalPods := 0 totalNodes := 0 From af994babcb49cb4627395a1e9a27297b79952dd4 Mon Sep 17 00:00:00 2001 From: Jonathan Innis Date: Fri, 31 Jan 2025 14:58:56 -0800 Subject: [PATCH 16/17] perf: Remove `Available()` call inside of filterInstanceTypesByRequirements (#1947) --- pkg/controllers/provisioning/scheduling/nodeclaim.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pkg/controllers/provisioning/scheduling/nodeclaim.go b/pkg/controllers/provisioning/scheduling/nodeclaim.go index db79322b36..6c380f0d5e 100644 --- a/pkg/controllers/provisioning/scheduling/nodeclaim.go +++ b/pkg/controllers/provisioning/scheduling/nodeclaim.go @@ -262,7 +262,16 @@ func filterInstanceTypesByRequirements(instanceTypes []*cloudprovider.InstanceTy // about why scheduling failed itCompat := compatible(it, requirements) itFits := fits(it, requests) - itHasOffering := it.Offerings.Available().HasCompatible(requirements) + + // By using this iterative approach vs. the Available() function it prevents allocations + // which have to be garbage collected and slow down Karpenter's scheduling algorithm + itHasOffering := false + for _, of := range it.Offerings { + if of.Available && requirements.IsCompatible(of.Requirements, scheduling.AllowUndefinedWellKnownLabels) { + itHasOffering = true + break + } + } // track if any single instance type met a single criteria results.requirementsMet = results.requirementsMet || itCompat From bb27849d7786397d5f6578a50a5192d4bbfa6117 Mon Sep 17 00:00:00 2001 From: Jonathan Innis Date: Fri, 31 Jan 2025 15:00:56 -0800 Subject: [PATCH 17/17] perf: Remove deep-copying the allocatable resource list (#1945) --- pkg/cloudprovider/types.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cloudprovider/types.go b/pkg/cloudprovider/types.go index 115c1afded..461c47bf7b 100644 --- a/pkg/cloudprovider/types.go +++ b/pkg/cloudprovider/types.go @@ -111,7 +111,7 @@ func (i *InstanceType) precompute() { func (i *InstanceType) Allocatable() corev1.ResourceList { i.once.Do(i.precompute) - return i.allocatable.DeepCopy() + return i.allocatable } func (its InstanceTypes) OrderByPrice(reqs scheduling.Requirements) InstanceTypes {