Skip to content

Commit

Permalink
add ut for loadaware scheduler plugin and npd plugin
Browse files Browse the repository at this point in the history
  • Loading branch information
WangZzzhe committed Jul 19, 2024
1 parent 0802c8a commit 4375250
Show file tree
Hide file tree
Showing 27 changed files with 2,373 additions and 141 deletions.
3 changes: 2 additions & 1 deletion cmd/katalyst-controller/app/options/npd.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,10 @@ limitations under the License.
package options

import (
cliflag "k8s.io/component-base/cli/flag"
"time"

cliflag "k8s.io/component-base/cli/flag"

"github.com/kubewharf/katalyst-core/pkg/config/controller"
)

Expand Down
2 changes: 1 addition & 1 deletion cmd/katalyst-scheduler/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,13 @@ limitations under the License.
package main

import (
"github.com/kubewharf/katalyst-core/pkg/scheduler/plugins/loadaware"
"os"

"github.com/spf13/cobra"
"k8s.io/component-base/logs"

"github.com/kubewharf/katalyst-core/cmd/katalyst-scheduler/app"
"github.com/kubewharf/katalyst-core/pkg/scheduler/plugins/loadaware"
"github.com/kubewharf/katalyst-core/pkg/scheduler/plugins/nodeovercommitment"
"github.com/kubewharf/katalyst-core/pkg/scheduler/plugins/noderesourcetopology"
"github.com/kubewharf/katalyst-core/pkg/scheduler/plugins/qosawarenoderesources"
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,7 @@ require (
)

replace (
github.com/kubewharf/katalyst-api => github.com/WangZzzhe/katalyst-api v0.0.0-20240626083651-4a90fe53af11
github.com/kubewharf/katalyst-api => github.com/WangZzzhe/katalyst-api v0.0.0-20240719035252-ac200da4db6c
k8s.io/api => k8s.io/api v0.24.6
k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.24.6
k8s.io/apimachinery => k8s.io/apimachinery v0.24.6
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -84,8 +84,8 @@ github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWX
github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g=
github.com/WangZzzhe/katalyst-api v0.0.0-20240626083651-4a90fe53af11 h1:4RUG7QfX0hBwtHtI3Nll6F4lCP31ThYxkWIu93G6Ei4=
github.com/WangZzzhe/katalyst-api v0.0.0-20240626083651-4a90fe53af11/go.mod h1:Y2IeIorxQamF2a3oa0+URztl5QCSty6Jj3zD83R8J9k=
github.com/WangZzzhe/katalyst-api v0.0.0-20240719035252-ac200da4db6c h1:/0fwVknrQEJoRKnT2H0f5xkzCdcDIH4qfNvpPn7QoH8=
github.com/WangZzzhe/katalyst-api v0.0.0-20240719035252-ac200da4db6c/go.mod h1:HHUJnOrDN5xrzKhEspq70ZJL859b09j07pMAl9ACnwU=
github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c=
github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
Expand Down
24 changes: 22 additions & 2 deletions pkg/controller/npd/indicator-plugin/loadaware/handler.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,19 @@
/*
Copyright 2022 The Katalyst Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package loadaware

import (
Expand Down Expand Up @@ -72,7 +88,9 @@ func (p *Plugin) OnPodAdd(obj interface{}) {
p.Lock()
defer p.Unlock()
if p.podUsageSelectorKey != "" {
if value, exist := pod.Labels[p.podUsageSelectorKey]; exist && value == p.podUsageSelectorVal {
if value, exist := pod.Labels[p.podUsageSelectorKey]; exist &&
value == p.podUsageSelectorVal &&
p.podUsageSelectorNamespace == pod.Namespace {
klog.Info("start sync pod usage to nodeMonitor")
p.enableSyncPodUsage = true
}
Expand Down Expand Up @@ -143,7 +161,9 @@ func (p *Plugin) OnPodDelete(obj interface{}) {
p.Lock()
defer p.Unlock()
if p.podUsageSelectorVal != "" {
if value, exist := pod.Labels[p.podUsageSelectorKey]; exist && value == p.podUsageSelectorVal {
if value, exist := pod.Labels[p.podUsageSelectorKey]; exist &&
value == p.podUsageSelectorVal &&
p.podUsageSelectorNamespace == pod.Namespace {
klog.Info("stop sync pod usage to nodeMonitor")
p.enableSyncPodUsage = false
}
Expand Down
143 changes: 143 additions & 0 deletions pkg/controller/npd/indicator-plugin/loadaware/handler_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
/*
Copyright 2022 The Katalyst Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package loadaware

import (
"testing"

"github.com/stretchr/testify/assert"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/sets"
)

func TestOnNodeAdd(t *testing.T) {
t.Parallel()

p := &Plugin{
workers: 3,
nodePoolMap: map[int32]sets.String{},
nodeStatDataMap: map[string]*NodeMetricData{},
}

testNode1 := &v1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: "testNode1",
},
Status: v1.NodeStatus{
Allocatable: map[v1.ResourceName]resource.Quantity{
v1.ResourceCPU: resource.MustParse("16"),
v1.ResourceMemory: resource.MustParse("32Gi"),
},
},
}

p.OnNodeAdd(testNode1)
assert.NotNil(t, p.nodeStatDataMap["testNode1"])
assert.Equal(t, 2, len(p.nodeStatDataMap["testNode1"].TotalRes))

p.OnNodeDelete(testNode1)
assert.Nil(t, p.nodeStatDataMap["testNode1"])
}

func TestOnNodeUpdate(t *testing.T) {
t.Parallel()

p := &Plugin{
nodeStatDataMap: map[string]*NodeMetricData{},
}

testNode1 := &v1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: "testNode1",
},
Status: v1.NodeStatus{
Allocatable: map[v1.ResourceName]resource.Quantity{
v1.ResourceCPU: resource.MustParse("16"),
v1.ResourceMemory: resource.MustParse("32Gi"),
},
},
}

p.OnNodeUpdate(nil, testNode1)
assert.NotNil(t, p.nodeStatDataMap["testNode1"])
assert.Equal(t, 2, len(p.nodeStatDataMap["testNode1"].TotalRes))
}

func TestOnPodAdd(t *testing.T) {
t.Parallel()

p := &Plugin{
nodeToPodsMap: map[string]map[string]struct{}{},
podUsageSelectorKey: "app",
podUsageSelectorVal: "testPod",
podUsageSelectorNamespace: "katalyst-system",
}

testPod1 := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "testPod1",
Namespace: "katalyst-system",
Labels: map[string]string{
"app": "testPod",
},
},
Spec: v1.PodSpec{
NodeName: "testNode1",
},
}

p.OnPodAdd(testPod1)
assert.NotNil(t, p.nodeToPodsMap["testNode1"])
assert.Equal(t, 1, len(p.nodeToPodsMap["testNode1"]))

p.OnPodDelete(testPod1)
assert.Equal(t, 0, len(p.nodeToPodsMap["testNode1"]))

p.OnPodDelete("")
}

func TestOnPodUpdate(t *testing.T) {
t.Parallel()

p := &Plugin{
nodeToPodsMap: map[string]map[string]struct{}{},
podUsageSelectorKey: "app",
podUsageSelectorVal: "testPod",
podUsageSelectorNamespace: "katalyst-system",
}

testPod1 := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "testPod1",
Namespace: "katalyst-system",
Labels: map[string]string{
"app": "testPod",
},
},
Spec: v1.PodSpec{
NodeName: "testNode1",
},
}

p.OnPodUpdate(nil, testPod1)
assert.NotNil(t, p.nodeToPodsMap["testNode1"])
assert.Equal(t, 1, len(p.nodeToPodsMap["testNode1"]))

p.OnPodUpdate(nil, "")
}
48 changes: 22 additions & 26 deletions pkg/controller/npd/indicator-plugin/loadaware/helper.go
Original file line number Diff line number Diff line change
@@ -1,13 +1,31 @@
/*
Copyright 2022 The Katalyst Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package loadaware

import (
"github.com/kubewharf/katalyst-core/pkg/controller/npd/indicator-plugin/loadaware/sorter"
"time"

corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
quotav1 "k8s.io/apiserver/pkg/quota/v1"
"k8s.io/metrics/pkg/apis/metrics/v1beta1"
"time"

"github.com/kubewharf/katalyst-core/pkg/controller/npd/indicator-plugin/loadaware/sorter"
)

// getUsage transfer cpu Nano to Milli, memory Ki to Mega
Expand Down Expand Up @@ -92,7 +110,7 @@ func refreshNodeMetricData(metricData *NodeMetricData, metricInfo *v1beta1.NodeM
max1Hour := calCPUAndMemoryMax(metricData.Latest1HourCache)
metricData.Max1Hour = max1Hour.DeepCopy()

//calculate 1 day max data
// calculate 1 day max data
if metricData.ifCanInsertLatest1DayCache(now) {
resWithTime := &ResourceListWithTime{
ResourceList: max1Hour.DeepCopy(),
Expand All @@ -115,7 +133,7 @@ func refreshPodMetricData(metricData *PodMetricData, metricInfo *v1beta1.PodMetr
podUsage = quotav1.Add(podUsage, containerMetrics.Usage)
}
metricData.LatestUsage = podUsage.DeepCopy()
//calculate 5 min avg data
// calculate 5 min avg data
metricData.Latest5MinCache = append(metricData.Latest5MinCache, getUsage(podUsage))
if len(metricData.Latest5MinCache) > Avg5MinPointNumber {
metricData.Latest5MinCache = metricData.Latest5MinCache[len(metricData.Latest5MinCache)-Avg5MinPointNumber:]
Expand Down Expand Up @@ -153,25 +171,3 @@ func getTopNPodUsages(podUsages map[string]corev1.ResourceList, maxPodUsageCount
}
return topNPodUsages
}

func calNodeLoad(resourceName corev1.ResourceName, usage, totalRes corev1.ResourceList) int64 {
if usage == nil || totalRes == nil {
return 0
}
used := int64(0)
total := int64(0)
if resourceName == corev1.ResourceCPU {
used = usage.Cpu().MilliValue()
total = totalRes.Cpu().MilliValue()
} else {
used = usage.Memory().Value()
total = totalRes.Memory().Value()
}
if total == 0 {
return 0
}
if used >= total {
return 99
}
return used * 100 / total
}
68 changes: 68 additions & 0 deletions pkg/controller/npd/indicator-plugin/loadaware/helper_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
/*
Copyright 2022 The Katalyst Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package loadaware

import (
"fmt"
"testing"

"github.com/stretchr/testify/assert"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
)

func TestGetTopNPodUsages(t *testing.T) {
t.Parallel()
podRealUsage := map[string]corev1.ResourceList{
"default/test-1": {
corev1.ResourceCPU: resource.MustParse("80"),
corev1.ResourceMemory: resource.MustParse("10Gi"),
},
"default/test-2": {
corev1.ResourceCPU: resource.MustParse("30"),
corev1.ResourceMemory: resource.MustParse("10Gi"),
},
"default/test-3": {
corev1.ResourceCPU: resource.MustParse("50"),
corev1.ResourceMemory: resource.MustParse("10Gi"),
},
"default/test-4": {
corev1.ResourceCPU: resource.MustParse("70"),
corev1.ResourceMemory: resource.MustParse("10Gi"),
},
"default/test-5": {
corev1.ResourceCPU: resource.MustParse("10"),
corev1.ResourceMemory: resource.MustParse("10Gi"),
},
"default/test-6": {
corev1.ResourceCPU: resource.MustParse("40"),
corev1.ResourceMemory: resource.MustParse("10Gi"),
},
"default/test-7": {
corev1.ResourceCPU: resource.MustParse("60"),
corev1.ResourceMemory: resource.MustParse("10Gi"),
},
}
resultMap := getTopNPodUsages(podRealUsage, 3)
expected := []string{"default/test-1", "default/test-4", "default/test-7"}
assert.Equal(t, len(resultMap), 3)
for _, v := range expected {
if _, ok := resultMap[v]; !ok {
t.Error(fmt.Errorf("not exit"))
}
}
}
Loading

0 comments on commit 4375250

Please sign in to comment.