From 8f4600f06f967350f57d68d63654bd3a92edcee0 Mon Sep 17 00:00:00 2001 From: "wangzhe.21" Date: Wed, 5 Jun 2024 10:43:18 +0800 Subject: [PATCH] feat: support allocate dedicated without exclusive and reclaim on a numa node --- .../dynamicpolicy/policy_advisor_handler.go | 43 +- .../policy_allocation_handlers.go | 14 +- .../cpu/dynamicpolicy/policy_test.go | 721 +++++++++++++++++- .../cpu/dynamicpolicy/state/state.go | 33 + .../cpu/dynamicpolicy/state/state_test.go | 107 +++ .../qosaware/resource/cpu/advisor_helper.go | 29 +- .../qosaware/resource/cpu/advisor_test.go | 81 +- .../provisionassembler/assembler_common.go | 53 +- .../provisionpolicy/policy_canonical.go | 8 +- .../provisionpolicy/policy_canonical_test.go | 90 ++- .../region/region_dedicated_numa_exclusive.go | 51 +- .../plugin/qosaware/server/cpu_server.go | 4 +- .../plugin/qosaware/server/cpu_server_test.go | 30 +- 13 files changed, 1191 insertions(+), 73 deletions(-) diff --git a/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy_advisor_handler.go b/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy_advisor_handler.go index b7835e871..fa8a35825 100644 --- a/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy_advisor_handler.go +++ b/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy_advisor_handler.go @@ -26,6 +26,7 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/status" + "k8s.io/klog/v2" pluginapi "k8s.io/kubelet/pkg/apis/resourceplugin/v1alpha1" maputil "k8s.io/kubernetes/pkg/util/maps" @@ -36,10 +37,12 @@ import ( advisorapi "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/cpu/dynamicpolicy/cpuadvisor" "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/cpu/dynamicpolicy/state" "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/util" + "github.com/kubewharf/katalyst-core/pkg/config/generic" "github.com/kubewharf/katalyst-core/pkg/metrics" "github.com/kubewharf/katalyst-core/pkg/util/general" "github.com/kubewharf/katalyst-core/pkg/util/machine" "github.com/kubewharf/katalyst-core/pkg/util/process" + "github.com/kubewharf/katalyst-core/pkg/util/qos" ) const ( @@ -284,7 +287,7 @@ func (p *DynamicPolicy) allocateByCPUAdvisor(resp *advisorapi.ListAndWatchRespon // generateBlockCPUSet generates BlockCPUSet from cpu-advisor response // and the logic contains three main steps -// 1. handle blocks for static pools +// 1. handle blocks for static pools and pods(dedicated_cores with numa_binding without numa_exclusive) // 2. handle blocks with specified NUMA ids (probably be blocks for // numa_binding dedicated_cores containers and reclaimed_cores containers colocated with them) // 3. handle blocks without specified NUMA id (probably be blocks for @@ -322,6 +325,44 @@ func (p *DynamicPolicy) generateBlockCPUSet(resp *advisorapi.ListAndWatchRespons availableCPUs = availableCPUs.Difference(blockCPUSet[blockID]) } + // walk through static pods to reuse cpuset if exists. + qosConf := generic.NewQoSConfiguration() + for podUID, containerEntries := range p.state.GetPodEntries() { + if containerEntries.IsPoolEntry() { + continue + } + if containerEntries.GetMainContainerEntry().QoSLevel != consts.PodAnnotationQoSLevelDedicatedCores { + continue + } + pod, err := p.metaServer.GetPod(context.Background(), podUID) + if err != nil { + err = fmt.Errorf("getPod %s fail: %v", podUID, err) + return nil, err + } + if !qos.IsPodNumaBinding(qosConf, pod) { + continue + } + if qos.IsPodNumaExclusive(qosConf, pod) { + continue + } + + for containerName, allocationInfo := range containerEntries { + for numaID, cpuset := range allocationInfo.TopologyAwareAssignments { + blocks, ok := resp.GeEntryNUMABlocks(podUID, containerName, int64(numaID)) + if !ok || len(blocks) != 1 { + klog.Error(err) + return nil, fmt.Errorf("blocks of pod %v container %v numaID %v is invalid", pod.Name, containerName, numaID) + } + + blockID := blocks[0].BlockId + blockCPUSet[blockID] = cpuset.Clone() + availableCPUs = availableCPUs.Difference(blockCPUSet[blockID]) + + klog.V(6).Infof("pod %v container %v numaId %v reuse cpuset %v", pod.Name, containerName, numaID, blockCPUSet[blockID]) + } + } + } + // walk through all blocks with specified NUMA ids // for each block, add them into blockCPUSet (if not exist) and renew availableCPUs for numaID, blocks := range numaToBlocks { diff --git a/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy_allocation_handlers.go b/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy_allocation_handlers.go index 7fcdf1303..84ffd5bec 100644 --- a/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy_allocation_handlers.go +++ b/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy_allocation_handlers.go @@ -24,6 +24,7 @@ import ( "time" v1 "k8s.io/api/core/v1" + "k8s.io/klog/v2" pluginapi "k8s.io/kubelet/pkg/apis/resourceplugin/v1alpha1" apiconsts "github.com/kubewharf/katalyst-api/pkg/consts" @@ -597,7 +598,11 @@ func (p *DynamicPolicy) adjustPoolsAndIsolatedEntries(poolsQuantityMap map[strin ) error { availableCPUs := machineState.GetFilteredAvailableCPUSet(p.reservedCPUs, nil, state.CheckDedicatedNUMABinding) - poolsCPUSet, isolatedCPUSet, err := p.generatePoolsAndIsolation(poolsQuantityMap, isolatedQuantityMap, availableCPUs) + // availableCPUs only for reclaimed pool, all unused cpu on dedicated without exclusive NUMA + reclaimedAvailableCPUs := machineState.GetMatchedAvailableCPUSet(p.reservedCPUs, state.CheckDedicatedNUMABindingWithoutNUMAExclusive, state.CheckDedicatedNUMABindingWithoutNUMAExclusive) + klog.V(6).Infof("adjustPoolsAndIsolatedEntries availableCPUs: %v, reclaimedAvailableCPUs: %v", availableCPUs.String(), reclaimedAvailableCPUs.String()) + + poolsCPUSet, isolatedCPUSet, err := p.generatePoolsAndIsolation(poolsQuantityMap, isolatedQuantityMap, availableCPUs, reclaimedAvailableCPUs) if err != nil { return fmt.Errorf("generatePoolsAndIsolation failed with error: %v", err) } @@ -642,7 +647,7 @@ func (p *DynamicPolicy) reclaimOverlapNUMABinding(poolsCPUSet map[string]machine } for _, allocationInfo := range containerEntries { - if !(allocationInfo != nil && state.CheckDedicatedNUMABinding(allocationInfo) && allocationInfo.CheckMainContainer()) { + if !(allocationInfo != nil && state.CheckDedicatedNUMABindingWithNUMAExclusive(allocationInfo) && allocationInfo.CheckMainContainer()) { continue } else if allocationInfo.RampUp { general.Infof("dedicated numa_binding pod: %s/%s container: %s is in ramp up, not to overlap reclaim pool with it", @@ -862,7 +867,7 @@ func (p *DynamicPolicy) applyPoolsAndIsolatedInfo(poolsCPUSet map[string]machine // 2. use the left cores to allocate among different pools // 3. apportion to other pools if reclaimed is disabled func (p *DynamicPolicy) generatePoolsAndIsolation(poolsQuantityMap map[string]int, - isolatedQuantityMap map[string]map[string]int, availableCPUs machine.CPUSet) (poolsCPUSet map[string]machine.CPUSet, + isolatedQuantityMap map[string]map[string]int, availableCPUs machine.CPUSet, reclaimAvailableCPUS machine.CPUSet) (poolsCPUSet map[string]machine.CPUSet, isolatedCPUSet map[string]map[string]machine.CPUSet, err error, ) { // clear pool map with zero quantity @@ -995,6 +1000,9 @@ func (p *DynamicPolicy) generatePoolsAndIsolation(poolsQuantityMap map[string]in } poolsCPUSet[state.PoolNameReclaim] = poolsCPUSet[state.PoolNameReclaim].Union(availableCPUs) + // join reclaim pool with all unused cpu on dedicated without exclusive NUMA + // final cpuSet include reservedResourceForAllocated will be calculated by advisor soon if cpu advisor is enabled + poolsCPUSet[state.PoolNameReclaim] = poolsCPUSet[state.PoolNameReclaim].Union(reclaimAvailableCPUS) if poolsCPUSet[state.PoolNameReclaim].IsEmpty() { // for reclaimed pool, we must make them exist when the node isn't in hybrid mode even if cause overlap allAvailableCPUs := p.machineInfo.CPUDetails.CPUs().Difference(p.reservedCPUs) diff --git a/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy_test.go b/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy_test.go index c566878e2..f2e76b465 100644 --- a/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy_test.go +++ b/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy_test.go @@ -32,6 +32,7 @@ import ( "github.com/stretchr/testify/require" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/uuid" pluginapi "k8s.io/kubelet/pkg/apis/resourceplugin/v1alpha1" utilfs "k8s.io/kubernetes/pkg/util/filesystem" @@ -2213,8 +2214,9 @@ func TestAllocateByQoSAwareServerListAndWatchResp(t *testing.T) { consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, }, Annotations: map[string]string{ - consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, - consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + consts.PodAnnotationMemoryEnhancementNumaExclusive: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, }, QoSLevel: consts.PodAnnotationQoSLevelDedicatedCores, RequestQuantity: 2, @@ -2555,8 +2557,9 @@ func TestAllocateByQoSAwareServerListAndWatchResp(t *testing.T) { consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, }, Annotations: map[string]string{ - consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, - consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + consts.PodAnnotationMemoryEnhancementNumaExclusive: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, }, QoSLevel: consts.PodAnnotationQoSLevelDedicatedCores, RequestQuantity: 2, @@ -2618,8 +2621,9 @@ func TestAllocateByQoSAwareServerListAndWatchResp(t *testing.T) { consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, }, Annotations: map[string]string{ - consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, - consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + consts.PodAnnotationMemoryEnhancementNumaExclusive: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, }, QoSLevel: consts.PodAnnotationQoSLevelDedicatedCores, RequestQuantity: 2, @@ -2810,23 +2814,694 @@ func TestAllocateByQoSAwareServerListAndWatchResp(t *testing.T) { }, cpuTopology: cpuTopology, }, - } - - for i, tc := range testCases { - tmpDir, err := ioutil.TempDir("", fmt.Sprintf("checkpoint-TestAllocateByQoSAwareServerListAndWatchResp-%v", i)) - as.Nil(err) - - dynamicPolicy, err := getTestDynamicPolicyWithInitialization(tc.cpuTopology, tmpDir) - as.Nil(err) - - dynamicPolicy.dynamicConfig.GetDynamicConfiguration().EnableReclaim = true - - machineState, err := generateMachineStateFromPodEntries(tc.cpuTopology, tc.podEntries) - as.Nil(err) - - dynamicPolicy.state.SetPodEntries(tc.podEntries) - dynamicPolicy.state.SetMachineState(machineState) - dynamicPolicy.initReservePool() + { + description: "dedicated without exclusive and shared", + podEntries: state.PodEntries{ + pod1UID: state.ContainerEntries{ + testName: &state.AllocationInfo{ + PodUid: pod1UID, + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN.String(), + ContainerIndex: 0, + RampUp: false, + OwnerPoolName: state.PoolNameDedicated, + AllocationResult: machine.MustParse("8,9"), + OriginalAllocationResult: machine.MustParse("8,9"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(8, 9), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 1: machine.NewCPUSet(8, 9), + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true"}`, + }, + QoSLevel: consts.PodAnnotationQoSLevelDedicatedCores, + RequestQuantity: 2, + }, + }, + pod2UID: state.ContainerEntries{ + testName: &state.AllocationInfo{ + PodUid: pod2UID, + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN.String(), + ContainerIndex: 0, + RampUp: false, + OwnerPoolName: state.PoolNameDedicated, + AllocationResult: machine.MustParse("10,11"), + OriginalAllocationResult: machine.MustParse("10,11"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 1: machine.NewCPUSet(10, 11), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 1: machine.NewCPUSet(10, 11), + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true"}`, + }, + QoSLevel: consts.PodAnnotationQoSLevelDedicatedCores, + RequestQuantity: 2, + }, + }, + pod3UID: state.ContainerEntries{ + testName: &state.AllocationInfo{ + PodUid: pod3UID, + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN.String(), + ContainerIndex: 0, + RampUp: false, + OwnerPoolName: state.PoolNameReclaim, + AllocationResult: machine.NewCPUSet(12, 13, 14, 15), + OriginalAllocationResult: machine.NewCPUSet(12, 13, 14, 15), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 2: machine.NewCPUSet(12, 13), + 3: machine.NewCPUSet(14, 15), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 2: machine.NewCPUSet(12, 13), + 3: machine.NewCPUSet(14, 15), + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, + }, + QoSLevel: consts.PodAnnotationQoSLevelReclaimedCores, + RequestQuantity: 2, + }, + }, + pod4UID: state.ContainerEntries{ + testName: &state.AllocationInfo{ + PodUid: pod4UID, + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN.String(), + ContainerIndex: 0, + RampUp: false, + OwnerPoolName: state.PoolNameShare, + AllocationResult: machine.NewCPUSet(4, 5, 6, 7), + OriginalAllocationResult: machine.NewCPUSet(4, 5, 6, 7), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 2: machine.NewCPUSet(4, 5), + 3: machine.NewCPUSet(6, 7), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 2: machine.NewCPUSet(4, 5), + 3: machine.NewCPUSet(6, 7), + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + QoSLevel: consts.PodAnnotationQoSLevelSharedCores, + RequestQuantity: 2, + }, + }, + }, + lwResp: &advisorapi.ListAndWatchResponse{ + Entries: map[string]*advisorapi.CalculationEntries{ + state.PoolNameShare: { + Entries: map[string]*advisorapi.CalculationInfo{ + "": { + OwnerPoolName: state.PoolNameShare, + CalculationResultsByNumas: map[int64]*advisorapi.NumaCalculationResult{ + -1: { + Blocks: []*advisorapi.Block{ + { + BlockId: "1b7f4ee0-65af-41a4-bb38-5f1268fd2f66", + Result: 4, + }, + }, + }, + }, + }, + }, + }, + state.PoolNameReserve: { + Entries: map[string]*advisorapi.CalculationInfo{ + "": { + OwnerPoolName: state.PoolNameReserve, + CalculationResultsByNumas: map[int64]*advisorapi.NumaCalculationResult{ + -1: { + Blocks: []*advisorapi.Block{ + { + BlockId: "1b7f4ee0-65af-41a4-bb38-5f1268fd2f65", + Result: 2, + }, + }, + }, + }, + }, + }, + }, + state.PoolNameReclaim: { + Entries: map[string]*advisorapi.CalculationInfo{ + "": { + OwnerPoolName: state.PoolNameReclaim, + CalculationResultsByNumas: map[int64]*advisorapi.NumaCalculationResult{ + 0: { + Blocks: []*advisorapi.Block{ + { + BlockId: "1b7f4ee0-65af-41a4-bb38-5f1268fd2f30", + Result: 1, + }, + }, + }, + 1: { + Blocks: []*advisorapi.Block{ + { + BlockId: "1b7f4ee0-65af-41a4-bb38-5f1268fd2f31", + Result: 1, + }, + }, + }, + -1: { + Blocks: []*advisorapi.Block{ + { + BlockId: "1b7f4ee0-65af-41a4-bb38-5f1268fd2f32", + Result: 4, + }, + }, + }, + }, + }, + }, + }, + pod1UID: { + Entries: map[string]*advisorapi.CalculationInfo{ + testName: { + OwnerPoolName: state.PoolNameDedicated, + CalculationResultsByNumas: map[int64]*advisorapi.NumaCalculationResult{ + 0: { + Blocks: []*advisorapi.Block{ + { + BlockId: "1b7f4ee0-65af-41a4-bb38-5f1268fd1f28", + Result: 2, + }, + }, + }, + }, + }, + }, + }, + pod2UID: { + Entries: map[string]*advisorapi.CalculationInfo{ + testName: { + OwnerPoolName: state.PoolNameDedicated, + CalculationResultsByNumas: map[int64]*advisorapi.NumaCalculationResult{ + 1: { + Blocks: []*advisorapi.Block{ + { + BlockId: "1b7f4ee0-65af-41a4-bb38-5f1268fd1f29", + Result: 2, + }, + }, + }, + }, + }, + }, + }, + }, + }, + expectedPodEntries: state.PodEntries{ + pod1UID: state.ContainerEntries{ + testName: &state.AllocationInfo{ + PodUid: pod1UID, + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN.String(), + ContainerIndex: 0, + RampUp: false, + OwnerPoolName: state.PoolNameShare, + AllocationResult: machine.MustParse("8,9"), + OriginalAllocationResult: machine.MustParse("8,9"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(8, 9), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 1: machine.NewCPUSet(8, 9), + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + }, + QoSLevel: consts.PodAnnotationQoSLevelDedicatedCores, + RequestQuantity: 2, + }, + }, + pod2UID: state.ContainerEntries{ + testName: &state.AllocationInfo{ + PodUid: pod2UID, + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN.String(), + ContainerIndex: 0, + RampUp: false, + OwnerPoolName: state.PoolNameShare, + AllocationResult: machine.MustParse("10,11"), + OriginalAllocationResult: machine.MustParse("10,11"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 1: machine.NewCPUSet(10, 11), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 1: machine.NewCPUSet(10, 11), + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + }, + QoSLevel: consts.PodAnnotationQoSLevelDedicatedCores, + RequestQuantity: 2, + }, + }, + pod3UID: state.ContainerEntries{ + testName: &state.AllocationInfo{ + PodUid: pod3UID, + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN.String(), + ContainerIndex: 0, + RampUp: false, + OwnerPoolName: state.PoolNameReclaim, + AllocationResult: machine.NewCPUSet(1, 3, 12, 13, 14, 15), + OriginalAllocationResult: machine.NewCPUSet(1, 3, 12, 13, 14, 15), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(1), + 1: machine.NewCPUSet(3), + 2: machine.NewCPUSet(12, 13), + 3: machine.NewCPUSet(14, 15), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(1), + 1: machine.NewCPUSet(3), + 2: machine.NewCPUSet(12, 13), + 3: machine.NewCPUSet(14, 15), + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, + }, + QoSLevel: consts.PodAnnotationQoSLevelReclaimedCores, + RequestQuantity: 2, + }, + }, + pod4UID: state.ContainerEntries{ + testName: &state.AllocationInfo{ + PodUid: pod4UID, + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN.String(), + ContainerIndex: 0, + RampUp: false, + OwnerPoolName: state.PoolNameShare, + AllocationResult: machine.NewCPUSet(4, 5, 6, 7), + OriginalAllocationResult: machine.NewCPUSet(4, 5, 6, 7), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 2: machine.NewCPUSet(4, 5), + 3: machine.NewCPUSet(6, 7), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 2: machine.NewCPUSet(4, 5), + 3: machine.NewCPUSet(6, 7), + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + QoSLevel: consts.PodAnnotationQoSLevelSharedCores, + RequestQuantity: 2, + }, + }, + state.PoolNameReclaim: state.ContainerEntries{ + "": &state.AllocationInfo{ + PodUid: state.PoolNameReclaim, + OwnerPoolName: state.PoolNameReclaim, + AllocationResult: machine.MustParse("1,3,12,13,14,15"), + OriginalAllocationResult: machine.MustParse("1,3,12,13,14,15"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(1), + 1: machine.NewCPUSet(3), + 2: machine.NewCPUSet(12, 13), + 3: machine.NewCPUSet(14, 15), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(1), + 1: machine.NewCPUSet(3), + 2: machine.NewCPUSet(12, 13), + 3: machine.NewCPUSet(14, 15), + }, + }, + }, + state.PoolNameShare: state.ContainerEntries{ + "": &state.AllocationInfo{ + PodUid: state.PoolNameShare, + OwnerPoolName: state.PoolNameShare, + AllocationResult: machine.MustParse("4,5,6,7"), + OriginalAllocationResult: machine.MustParse("4,5,6,7"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 2: machine.NewCPUSet(4, 5), + 3: machine.NewCPUSet(6, 7), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 2: machine.NewCPUSet(4, 5), + 3: machine.NewCPUSet(6, 7), + }, + }, + }, + state.PoolNameReserve: state.ContainerEntries{ + "": &state.AllocationInfo{ + PodUid: state.PoolNameReserve, + OwnerPoolName: state.PoolNameReserve, + AllocationResult: machine.MustParse("0,2"), + OriginalAllocationResult: machine.MustParse("0,2"), + TopologyAwareAssignments: map[int]machine.CPUSet{}, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{}, + }, + }, + }, + expectedMachineState: state.NUMANodeMap{ + 0: &state.NUMANodeState{ + DefaultCPUSet: machine.NewCPUSet(0, 1), + AllocatedCPUSet: machine.NewCPUSet(8, 9), + PodEntries: state.PodEntries{ + pod1UID: state.ContainerEntries{ + testName: &state.AllocationInfo{ + PodUid: pod1UID, + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN.String(), + ContainerIndex: 0, + RampUp: false, + OwnerPoolName: state.PoolNameReclaim, + AllocationResult: machine.MustParse("8,9"), + OriginalAllocationResult: machine.MustParse("8,9"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(8, 9), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(8, 9), + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + }, + QoSLevel: consts.PodAnnotationQoSLevelDedicatedCores, + RequestQuantity: 2, + }, + }, + pod3UID: state.ContainerEntries{ + testName: &state.AllocationInfo{ + PodUid: pod4UID, + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN.String(), + ContainerIndex: 0, + RampUp: false, + OwnerPoolName: state.PoolNameReclaim, + AllocationResult: machine.NewCPUSet(1), + OriginalAllocationResult: machine.NewCPUSet(1), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(1), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(1), + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, + }, + QoSLevel: consts.PodAnnotationQoSLevelReclaimedCores, + RequestQuantity: 1, + }, + }, + }, + }, + 1: &state.NUMANodeState{ + DefaultCPUSet: machine.NewCPUSet(2, 3), + AllocatedCPUSet: machine.NewCPUSet(10, 11), + PodEntries: state.PodEntries{ + pod2UID: state.ContainerEntries{ + testName: &state.AllocationInfo{ + PodUid: pod3UID, + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN.String(), + ContainerIndex: 0, + RampUp: false, + OwnerPoolName: state.PoolNameReclaim, + AllocationResult: machine.MustParse("10,11"), + OriginalAllocationResult: machine.MustParse("10,11"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 1: machine.NewCPUSet(10, 11), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 1: machine.NewCPUSet(10, 11), + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + }, + QoSLevel: consts.PodAnnotationQoSLevelDedicatedCores, + RequestQuantity: 2, + }, + }, + pod3UID: state.ContainerEntries{ + testName: &state.AllocationInfo{ + PodUid: pod4UID, + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN.String(), + ContainerIndex: 0, + RampUp: false, + OwnerPoolName: state.PoolNameReclaim, + AllocationResult: machine.NewCPUSet(3), + OriginalAllocationResult: machine.NewCPUSet(3), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(3), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(3), + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, + }, + QoSLevel: consts.PodAnnotationQoSLevelReclaimedCores, + RequestQuantity: 1, + }, + }, + }, + }, + 2: &state.NUMANodeState{ + DefaultCPUSet: cpuTopology.CPUDetails.CPUsInNUMANodes(2).Clone(), + AllocatedCPUSet: machine.NewCPUSet(), + PodEntries: state.PodEntries{ + pod4UID: state.ContainerEntries{ + testName: &state.AllocationInfo{ + PodUid: pod2UID, + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN.String(), + ContainerIndex: 0, + RampUp: false, + OwnerPoolName: state.PoolNameShare, + AllocationResult: machine.MustParse("4,5"), + OriginalAllocationResult: machine.MustParse("4-5"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 2: machine.NewCPUSet(4, 5), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 2: machine.NewCPUSet(4, 5), + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + QoSLevel: consts.PodAnnotationQoSLevelSharedCores, + RequestQuantity: 2, + }, + }, + pod3UID: state.ContainerEntries{ + testName: &state.AllocationInfo{ + PodUid: pod3UID, + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN.String(), + ContainerIndex: 0, + RampUp: false, + OwnerPoolName: state.PoolNameReclaim, + AllocationResult: machine.MustParse("12,13"), + OriginalAllocationResult: machine.MustParse("12,13"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 2: machine.NewCPUSet(12, 13), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 2: machine.NewCPUSet(12, 13), + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, + }, + QoSLevel: consts.PodAnnotationQoSLevelReclaimedCores, + RequestQuantity: 2, + }, + }, + }, + }, + 3: &state.NUMANodeState{ + DefaultCPUSet: cpuTopology.CPUDetails.CPUsInNUMANodes(3).Clone(), + AllocatedCPUSet: machine.NewCPUSet(), + PodEntries: state.PodEntries{ + pod4UID: state.ContainerEntries{ + testName: &state.AllocationInfo{ + PodUid: pod1UID, + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN.String(), + ContainerIndex: 0, + RampUp: false, + OwnerPoolName: state.PoolNameShare, + AllocationResult: machine.MustParse("6,7"), + OriginalAllocationResult: machine.MustParse("6,7"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 3: machine.NewCPUSet(6, 7), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 3: machine.NewCPUSet(6, 7), + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + QoSLevel: consts.PodAnnotationQoSLevelSharedCores, + RequestQuantity: 2, + }, + }, + pod3UID: state.ContainerEntries{ + testName: &state.AllocationInfo{ + PodUid: pod2UID, + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN.String(), + ContainerIndex: 0, + RampUp: false, + OwnerPoolName: state.PoolNameReclaim, + AllocationResult: machine.MustParse("14,15"), + OriginalAllocationResult: machine.MustParse("14,15"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 3: machine.NewCPUSet(14, 15), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 3: machine.NewCPUSet(14, 15), + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, + }, + QoSLevel: consts.PodAnnotationQoSLevelReclaimedCores, + RequestQuantity: 2, + }, + }, + }, + }, + }, + cpuTopology: cpuTopology, + }, + } + + for i, tc := range testCases { + tmpDir, err := ioutil.TempDir("", fmt.Sprintf("checkpoint-TestAllocateByQoSAwareServerListAndWatchResp-%v", i)) + as.Nil(err) + + dynamicPolicy, err := getTestDynamicPolicyWithInitialization(tc.cpuTopology, tmpDir) + as.Nil(err) + + dynamicPolicy.dynamicConfig.GetDynamicConfiguration().EnableReclaim = true + + machineState, err := generateMachineStateFromPodEntries(tc.cpuTopology, tc.podEntries) + as.Nil(err) + + dynamicPolicy.state.SetPodEntries(tc.podEntries) + dynamicPolicy.state.SetMachineState(machineState) + dynamicPolicy.initReservePool() + + podFetcher := &pod.PodFetcherStub{ + PodList: make([]*v1.Pod, 0), + } + for _, podEntry := range tc.podEntries { + for _, containerEntry := range podEntry { + podFetcher.PodList = append(podFetcher.PodList, &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: containerEntry.PodName, + UID: types.UID(containerEntry.PodUid), + Annotations: containerEntry.Annotations, + Labels: containerEntry.Labels, + }, + }) + break + } + } + dynamicPolicy.metaServer.PodFetcher = podFetcher err = dynamicPolicy.allocateByCPUAdvisor(tc.lwResp) as.Nilf(err, "dynamicPolicy.allocateByCPUAdvisorServerListAndWatchResp got err: %v, case: %s", err, tc.description) diff --git a/pkg/agent/qrm-plugins/cpu/dynamicpolicy/state/state.go b/pkg/agent/qrm-plugins/cpu/dynamicpolicy/state/state.go index e082f1fcc..646e56b2b 100644 --- a/pkg/agent/qrm-plugins/cpu/dynamicpolicy/state/state.go +++ b/pkg/agent/qrm-plugins/cpu/dynamicpolicy/state/state.go @@ -196,12 +196,29 @@ func CheckNUMABinding(ai *AllocationInfo) bool { return ai.Annotations[consts.PodAnnotationMemoryEnhancementNumaBinding] == consts.PodAnnotationMemoryEnhancementNumaBindingEnable } +// CheckNUMAExclusive returns true if the AllocationInfo is for pod with numa-exclusive enhancement +func CheckNUMAExclusive(ai *AllocationInfo) bool { + return ai.Annotations[consts.PodAnnotationMemoryEnhancementNumaExclusive] == consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable +} + // CheckDedicatedNUMABinding returns true if the AllocationInfo is for pod with // dedicated-qos and numa-binding enhancement func CheckDedicatedNUMABinding(ai *AllocationInfo) bool { return CheckDedicated(ai) && CheckNUMABinding(ai) } +// CheckDedicatedNUMABindingWithNUMAExclusive returns true if the AllocationInfo is for pod with +// dedicated-qos and numa-binding and numa-exclusive enhancement +func CheckDedicatedNUMABindingWithNUMAExclusive(ai *AllocationInfo) bool { + return CheckDedicated(ai) && CheckNUMABinding(ai) && CheckNUMAExclusive(ai) +} + +// CheckDedicatedNUMABindingWithoutNUMAExclusive returns true if the AllocationInfo is for pod with +// dedicated-qos and numa-binding and without numa-exclusive enhancement +func CheckDedicatedNUMABindingWithoutNUMAExclusive(ai *AllocationInfo) bool { + return CheckDedicated(ai) && CheckNUMABinding(ai) && (!CheckNUMAExclusive(ai)) +} + // CheckDedicatedPool returns true if the AllocationInfo is for a container in the dedicated pool func CheckDedicatedPool(ai *AllocationInfo) bool { return ai.OwnerPoolName == PoolNameDedicated @@ -438,6 +455,22 @@ func (nm NUMANodeMap) GetFilteredAvailableCPUSet(reservedCPUs machine.CPUSet, return nm.GetFilteredDefaultCPUSet(excludeEntry, excludeWholeNUMA).Difference(reservedCPUs) } +// GetMatchedAvailableCPUSet returns available cpuset on includeNUMA, along with the excludeEntry filter functions +func (nm NUMANodeMap) GetMatchedAvailableCPUSet(reservedCPUs machine.CPUSet, + includeNUMA, excludeEntry func(ai *AllocationInfo) bool, +) machine.CPUSet { + res := machine.NewCPUSet() + for _, numaNodeState := range nm { + if !numaNodeState.ExistMatchedAllocationInfo(includeNUMA) { + continue + } + + res = res.Union(numaNodeState.GetFilteredDefaultCPUSet(excludeEntry, nil)) + } + + return res.Difference(reservedCPUs) +} + // GetFilteredNUMASet return numa set except the numa which are excluded by the predicate. func (nm NUMANodeMap) GetFilteredNUMASet(excludeNUMAPredicate func(ai *AllocationInfo) bool) machine.CPUSet { res := machine.NewCPUSet() diff --git a/pkg/agent/qrm-plugins/cpu/dynamicpolicy/state/state_test.go b/pkg/agent/qrm-plugins/cpu/dynamicpolicy/state/state_test.go index b60595e43..962eb7df9 100644 --- a/pkg/agent/qrm-plugins/cpu/dynamicpolicy/state/state_test.go +++ b/pkg/agent/qrm-plugins/cpu/dynamicpolicy/state/state_test.go @@ -2539,3 +2539,110 @@ func TestGetSocketTopology(t *testing.T) { as.Equalf(tc.expectedSocketTopology, actualSocketToplogy, "failed in test case: %s", tc.description) } } + +func TestCheckNUMAExclusive(t *testing.T) { + t.Parallel() + + ai := &AllocationInfo{ + Annotations: map[string]string{ + consts.PodAnnotationMemoryEnhancementNumaExclusive: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, + }, + } + assert.True(t, CheckNUMAExclusive(ai)) + + ai = &AllocationInfo{ + Annotations: map[string]string{ + consts.PodAnnotationMemoryEnhancementNumaExclusive: "", + }, + } + assert.False(t, CheckNUMAExclusive(ai)) + + ai = &AllocationInfo{ + Annotations: map[string]string{ + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, + }, + } + assert.False(t, CheckNUMAExclusive(ai)) +} + +func TestCheckDedicatedNUMABindingWithNUMAExclusive(t *testing.T) { + t.Parallel() + + ai := &AllocationInfo{ + Annotations: map[string]string{ + consts.PodAnnotationMemoryEnhancementNumaExclusive: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + }, + QoSLevel: consts.PodAnnotationQoSLevelDedicatedCores, + } + assert.True(t, CheckDedicatedNUMABindingWithNUMAExclusive(ai)) + + ai = &AllocationInfo{ + Annotations: map[string]string{ + consts.PodAnnotationMemoryEnhancementNumaExclusive: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + }, + QoSLevel: consts.PodAnnotationQoSLevelSharedCores, + } + assert.False(t, CheckDedicatedNUMABindingWithNUMAExclusive(ai)) +} + +func TestCheckDedicatedNUMABindingWithoutNUMAExclusive(t *testing.T) { + t.Parallel() + + ai := &AllocationInfo{ + Annotations: map[string]string{ + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + }, + QoSLevel: consts.PodAnnotationQoSLevelDedicatedCores, + } + assert.True(t, CheckDedicatedNUMABindingWithoutNUMAExclusive(ai)) + + ai = &AllocationInfo{ + Annotations: map[string]string{ + consts.PodAnnotationMemoryEnhancementNumaExclusive: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + }, + QoSLevel: consts.PodAnnotationQoSLevelDedicatedCores, + } + assert.False(t, CheckDedicatedNUMABindingWithoutNUMAExclusive(ai)) + + ai = &AllocationInfo{ + Annotations: map[string]string{ + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + }, + QoSLevel: consts.PodAnnotationQoSLevelSharedCores, + } + assert.False(t, CheckDedicatedNUMABindingWithoutNUMAExclusive(ai)) +} + +func TestGetMatchedAvailableCPUSet(t *testing.T) { + t.Parallel() + + ns := map[int]*NUMANodeState{ + 0: { + DefaultCPUSet: machine.NewCPUSet(0, 1, 2, 3, 4, 5, 6, 7), + AllocatedCPUSet: machine.NewCPUSet(), + }, + 1: { + DefaultCPUSet: machine.NewCPUSet(12, 13, 14, 15), + AllocatedCPUSet: machine.NewCPUSet(8, 9, 10, 11), + PodEntries: map[string]ContainerEntries{ + "podUID": map[string]*AllocationInfo{ + "testContainer": { + QoSLevel: consts.PodAnnotationQoSLevelDedicatedCores, + Annotations: map[string]string{ + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + }, + AllocationResult: machine.NewCPUSet(8, 9, 10, 11), + }, + }, + }, + }, + } + nm := NUMANodeMap(ns) + + reservedCPUs := machine.NewCPUSet(0, 1) + res := nm.GetMatchedAvailableCPUSet(reservedCPUs, CheckDedicatedNUMABindingWithoutNUMAExclusive, CheckDedicatedNUMABindingWithoutNUMAExclusive) + assert.True(t, res.Equals(machine.NewCPUSet(12, 13, 14, 15))) +} diff --git a/pkg/agent/sysadvisor/plugin/qosaware/resource/cpu/advisor_helper.go b/pkg/agent/sysadvisor/plugin/qosaware/resource/cpu/advisor_helper.go index f24c03a15..e9fd5c4ce 100644 --- a/pkg/agent/sysadvisor/plugin/qosaware/resource/cpu/advisor_helper.go +++ b/pkg/agent/sysadvisor/plugin/qosaware/resource/cpu/advisor_helper.go @@ -183,6 +183,22 @@ func (cra *cpuResourceAdvisor) getRegionMaxRequirement(r region.QoSRegion) float return true }) res = general.MaxFloat64(1, res) + + case types.QoSRegionTypeDedicatedNumaExclusive: + dr := r.(*region.QoSRegionDedicatedNumaExclusive) + if !dr.IsNumaExclusive() { + request, err := dr.GetRegionRequest() + if err != nil { + klog.Errorf("region %v GetRegionRequest fail: %v, use MinDedicatedCPURequirement %v instead", + r.Name(), err, types.MinDedicatedCPURequirement) + res = types.MinDedicatedCPURequirement + } else { + res = request + } + return res + } + fallthrough + default: for _, numaID := range r.GetBindingNumas().ToSliceInt() { res += float64(cra.numaAvailable[numaID]) @@ -208,7 +224,18 @@ func (cra *cpuResourceAdvisor) getRegionMinRequirement(r region.QoSRegion) float res = general.MaxFloat64(1, res) return res case types.QoSRegionTypeDedicatedNumaExclusive: - return types.MinDedicatedCPURequirement + dr := r.(*region.QoSRegionDedicatedNumaExclusive) + if dr.IsNumaExclusive() { + return types.MinDedicatedCPURequirement + } else { + request, err := dr.GetRegionRequest() + if err != nil { + klog.Errorf("region %v GetRegionRequest fail: %v, use MinDedicatedCPURequirement %v instead", + r.Name(), err, types.MinDedicatedCPURequirement) + return types.MinDedicatedCPURequirement + } + return request + } default: klog.Errorf("[qosaware-cpu] unknown region type %v", r.Type()) return 0.0 diff --git a/pkg/agent/sysadvisor/plugin/qosaware/resource/cpu/advisor_test.go b/pkg/agent/sysadvisor/plugin/qosaware/resource/cpu/advisor_test.go index e824a1ca9..653f2d1e5 100644 --- a/pkg/agent/sysadvisor/plugin/qosaware/resource/cpu/advisor_test.go +++ b/pkg/agent/sysadvisor/plugin/qosaware/resource/cpu/advisor_test.go @@ -432,7 +432,10 @@ func TestAdvisorUpdate(t *testing.T) { }, containers: []*types.ContainerInfo{ makeContainerInfo("uid1", "default", "pod1", "c1", consts.PodAnnotationQoSLevelDedicatedCores, state.PoolNameDedicated, - map[string]string{consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable}, + map[string]string{ + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + consts.PodAnnotationMemoryEnhancementNumaExclusive: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, + }, map[int]machine.CPUSet{ 0: machine.MustParse("1-23,48-71"), }, 36), @@ -482,7 +485,10 @@ func TestAdvisorUpdate(t *testing.T) { }, containers: []*types.ContainerInfo{ makeContainerInfo("uid1", "default", "pod1", "c1", consts.PodAnnotationQoSLevelDedicatedCores, state.PoolNameDedicated, - map[string]string{consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable}, + map[string]string{ + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + consts.PodAnnotationMemoryEnhancementNumaExclusive: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, + }, map[int]machine.CPUSet{ 0: machine.MustParse("1-23,48-71"), }, 36), @@ -536,7 +542,10 @@ func TestAdvisorUpdate(t *testing.T) { }, containers: []*types.ContainerInfo{ makeContainerInfo("uid1", "default", "pod1", "c1", consts.PodAnnotationQoSLevelDedicatedCores, state.PoolNameDedicated, - map[string]string{consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable}, + map[string]string{ + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + consts.PodAnnotationMemoryEnhancementNumaExclusive: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, + }, map[int]machine.CPUSet{ 0: machine.MustParse("1-23,48-71"), }, 36), @@ -590,7 +599,10 @@ func TestAdvisorUpdate(t *testing.T) { }, containers: []*types.ContainerInfo{ makeContainerInfo("uid1", "default", "pod1", "c1", consts.PodAnnotationQoSLevelDedicatedCores, state.PoolNameDedicated, - map[string]string{consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable}, + map[string]string{ + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + consts.PodAnnotationMemoryEnhancementNumaExclusive: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, + }, map[int]machine.CPUSet{ 0: machine.MustParse("1-23,48-71"), }, 36), @@ -650,7 +662,10 @@ func TestAdvisorUpdate(t *testing.T) { }, containers: []*types.ContainerInfo{ makeContainerInfo("uid1", "default", "pod1", "c1", consts.PodAnnotationQoSLevelDedicatedCores, state.PoolNameDedicated, - map[string]string{consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable}, + map[string]string{ + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + consts.PodAnnotationMemoryEnhancementNumaExclusive: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, + }, map[int]machine.CPUSet{ 0: machine.MustParse("1-23,48-71"), }, 36), @@ -718,7 +733,10 @@ func TestAdvisorUpdate(t *testing.T) { }, containers: []*types.ContainerInfo{ makeContainerInfo("uid1", "default", "pod1", "c1", consts.PodAnnotationQoSLevelDedicatedCores, state.PoolNameDedicated, - map[string]string{consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable}, + map[string]string{ + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + consts.PodAnnotationMemoryEnhancementNumaExclusive: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, + }, map[int]machine.CPUSet{ 0: machine.MustParse("1-23,48-71"), }, 36), @@ -1109,6 +1127,57 @@ func TestAdvisorUpdate(t *testing.T) { }, }, }, + { + name: "provision:dedicated_numa_without_exclusive", + pools: map[string]*types.PoolInfo{ + state.PoolNameReserve: { + PoolName: state.PoolNameReserve, + TopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.MustParse("0"), + 1: machine.MustParse("24"), + }, + }, + state.PoolNameReclaim: { + PoolName: state.PoolNameReclaim, + TopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.MustParse("70-71"), + 1: machine.MustParse("25-46"), + }, + }, + }, + containers: []*types.ContainerInfo{ + makeContainerInfo("uid1", "default", "pod1", "c1", consts.PodAnnotationQoSLevelDedicatedCores, state.PoolNameDedicated, + map[string]string{ + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + }, + map[int]machine.CPUSet{ + 0: machine.MustParse("1-23,48"), + }, 24), + }, + pods: []*v1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "default", + UID: "uid1", + }, + }, + }, + nodeEnableReclaim: true, + headroomAssembler: types.CPUHeadroomAssemblerCommon, + wantInternalCalculationResult: types.InternalCPUCalculationResult{ + PoolEntries: map[string]map[int]int{ + state.PoolNameReserve: { + -1: 2, + }, + state.PoolNameReclaim: { + 0: 21, + -1: 47, + }, + }, + }, + wantHeadroom: resource.Quantity{}, + }, } for _, tt := range tests { diff --git a/pkg/agent/sysadvisor/plugin/qosaware/resource/cpu/assembler/provisionassembler/assembler_common.go b/pkg/agent/sysadvisor/plugin/qosaware/resource/cpu/assembler/provisionassembler/assembler_common.go index a2819c105..a7096c569 100644 --- a/pkg/agent/sysadvisor/plugin/qosaware/resource/cpu/assembler/provisionassembler/assembler_common.go +++ b/pkg/agent/sysadvisor/plugin/qosaware/resource/cpu/assembler/provisionassembler/assembler_common.go @@ -19,8 +19,10 @@ package provisionassembler import ( "context" "fmt" + "math" "time" + v1 "k8s.io/api/core/v1" "k8s.io/klog/v2" "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/cpu/dynamicpolicy/cpuadvisor" @@ -79,6 +81,7 @@ func (pa *ProvisionAssemblerCommon) AssembleProvision() (types.InternalCPUCalcul shares := 0 isolationUppers := 0 + dedicatedNonReclaimCpuSize := make(map[int]int) sharePoolSizes := make(map[string]int) isolationUpperSizes := make(map[string]int) @@ -119,21 +122,49 @@ func (pa *ProvisionAssemblerCommon) AssembleProvision() (types.InternalCPUCalcul return types.InternalCPUCalculationResult{}, err } - // fill in reclaim pool entry for dedicated numa exclusive regions - if !enableReclaim { - if reservedForReclaim > 0 { - calculationResult.SetPoolEntry(state.PoolNameReclaim, regionNuma, reservedForReclaim) + dr := r.(*region.QoSRegionDedicatedNumaExclusive) + + if dr.IsNumaExclusive() { + // fill in reclaim pool entry for dedicated numa exclusive regions + if !enableReclaim { + if reservedForReclaim > 0 { + calculationResult.SetPoolEntry(state.PoolNameReclaim, regionNuma, reservedForReclaim) + } + } else { + available := getNumasAvailableResource(*pa.numaAvailable, r.GetBindingNumas()) + nonReclaimRequirement := int(controlKnob[types.ControlKnobNonReclaimedCPUSize].Value) + reclaimed := available - nonReclaimRequirement + reservedForReclaim + + calculationResult.SetPoolEntry(state.PoolNameReclaim, regionNuma, reclaimed) + + klog.InfoS("assemble info", "regionName", r.Name(), "reclaimed", reclaimed, + "available", available, "nonReclaimRequirement", nonReclaimRequirement, "reservedForReclaim", reservedForReclaim) } } else { - available := getNumasAvailableResource(*pa.numaAvailable, r.GetBindingNumas()) - nonReclaimRequirement := int(controlKnob[types.ControlKnobNonReclaimedCPUSize].Value) - reclaimed := available - nonReclaimRequirement + reservedForReclaim + dedicatedNonReclaimCpuSize[regionNuma] += int(controlKnob[types.ControlKnobNonReclaimedCPUSize].Value) + } + } + } + + var ( + reservedPerNuma int + reservedForAllocate = pa.conf.GetDynamicConfiguration().ReservedResourceForAllocate[v1.ResourceCPU] + ) + if len(dedicatedNonReclaimCpuSize) > 0 { + reservedPerNuma = int(math.Ceil(float64(reservedForAllocate.Value()*int64(len(dedicatedNonReclaimCpuSize))) / + float64(pa.metaServer.NumNUMANodes))) + } + for numaID, cpuSize := range dedicatedNonReclaimCpuSize { + numas := machine.NewCPUSet(numaID) + reservedForReclaim := pa.getNumasReservedForReclaim(numas) + available := getNumasAvailableResource(*pa.numaAvailable, numas) + reclaimed := available - cpuSize - reservedPerNuma + reservedForReclaim - calculationResult.SetPoolEntry(state.PoolNameReclaim, regionNuma, reclaimed) + klog.InfoS("assemble info", "dedicated numaID", numaID, "reclaimed", reclaimed, + "availabel", available, "nonReclaimRequirement", cpuSize, "reservedPerNuma", reservedPerNuma, "reservedForReclaim", reservedForReclaim) - klog.InfoS("assemble info", "regionName", r.Name(), "reclaimed", reclaimed, - "available", available, "nonReclaimRequirement", nonReclaimRequirement, "reservedForReclaim", reservedForReclaim) - } + if reclaimed > 0 { + calculationResult.SetPoolEntry(state.PoolNameReclaim, numaID, reclaimed) } } diff --git a/pkg/agent/sysadvisor/plugin/qosaware/resource/cpu/region/provisionpolicy/policy_canonical.go b/pkg/agent/sysadvisor/plugin/qosaware/resource/cpu/region/provisionpolicy/policy_canonical.go index 1145d1034..71a784b2d 100644 --- a/pkg/agent/sysadvisor/plugin/qosaware/resource/cpu/region/provisionpolicy/policy_canonical.go +++ b/pkg/agent/sysadvisor/plugin/qosaware/resource/cpu/region/provisionpolicy/policy_canonical.go @@ -106,7 +106,13 @@ func (p *PolicyCanonical) estimateCPUUsage() (float64, error) { } var containerEstimation float64 = 0 - if ci.IsNumaBinding() && !enableReclaim { + if ci.IsNumaBinding() && !ci.IsNumaExclusive() { + if ci.ContainerType == v1alpha1.ContainerType_MAIN { + containerEstimation += ci.CPURequest + } else { + containerEstimation = 0 + } + } else if ci.IsNumaBinding() && !enableReclaim { if ci.ContainerType == v1alpha1.ContainerType_MAIN { bindingNumas := machine.GetCPUAssignmentNUMAs(ci.TopologyAwareAssignments) for range bindingNumas.ToSliceInt() { diff --git a/pkg/agent/sysadvisor/plugin/qosaware/resource/cpu/region/provisionpolicy/policy_canonical_test.go b/pkg/agent/sysadvisor/plugin/qosaware/resource/cpu/region/provisionpolicy/policy_canonical_test.go index 9aff065d7..c2e8a6a7c 100644 --- a/pkg/agent/sysadvisor/plugin/qosaware/resource/cpu/region/provisionpolicy/policy_canonical_test.go +++ b/pkg/agent/sysadvisor/plugin/qosaware/resource/cpu/region/provisionpolicy/policy_canonical_test.go @@ -26,6 +26,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" k8types "k8s.io/apimachinery/pkg/types" + "k8s.io/kubelet/pkg/apis/resourceplugin/v1alpha1" apiconsts "github.com/kubewharf/katalyst-api/pkg/consts" katalyst_base "github.com/kubewharf/katalyst-core/cmd/base" @@ -157,7 +158,7 @@ func TestPolicyCanonical(t *testing.T) { tests := []struct { name string regionInfo types.RegionInfo - containerInfo map[string]map[string]types.ContainerInfo + containerInfo map[string]map[string]*types.ContainerInfo containerMetricData map[string]map[string]map[string]metricutil.MetricData resourceEssentials types.ResourceEssentials controlEssentials types.ControlEssentials @@ -165,9 +166,9 @@ func TestPolicyCanonical(t *testing.T) { }{ { name: "share_ramp_up", - containerInfo: map[string]map[string]types.ContainerInfo{ + containerInfo: map[string]map[string]*types.ContainerInfo{ "pod0": { - "container0": types.ContainerInfo{ + "container0": &types.ContainerInfo{ PodUID: "pod0", PodName: "pod0", ContainerName: "container0", @@ -220,9 +221,9 @@ func TestPolicyCanonical(t *testing.T) { }, { name: "share_ramp_down", - containerInfo: map[string]map[string]types.ContainerInfo{ + containerInfo: map[string]map[string]*types.ContainerInfo{ "pod0": { - "container0": types.ContainerInfo{ + "container0": &types.ContainerInfo{ PodUID: "pod0", PodName: "pod0", ContainerName: "container0", @@ -275,9 +276,9 @@ func TestPolicyCanonical(t *testing.T) { }, { name: "dedicated_numa_exclusive", - containerInfo: map[string]map[string]types.ContainerInfo{ + containerInfo: map[string]map[string]*types.ContainerInfo{ "pod0": { - "container0": types.ContainerInfo{ + "container0": &types.ContainerInfo{ PodUID: "pod0", PodName: "pod0", ContainerName: "container0", @@ -327,6 +328,79 @@ func TestPolicyCanonical(t *testing.T) { }, }, }, + { + name: "dedicated_numa_without_exclusive", + containerInfo: map[string]map[string]*types.ContainerInfo{ + "testpod": { + "container0": &types.ContainerInfo{ + ContainerType: v1alpha1.ContainerType_MAIN, + PodUID: "pod0", + PodName: "pod0", + ContainerName: "container0", + QoSLevel: apiconsts.PodAnnotationQoSLevelDedicatedCores, + Annotations: map[string]string{ + apiconsts.PodAnnotationMemoryEnhancementNumaBinding: apiconsts.PodAnnotationMemoryEnhancementNumaBindingEnable, + }, + CPURequest: 4.0, + RampUp: false, + TopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(0, 1, 2, 3), + }, + }, + "container1": &types.ContainerInfo{ + ContainerType: v1alpha1.ContainerType_SIDECAR, + PodUID: "pod0", + PodName: "pod0", + ContainerName: "container1", + QoSLevel: apiconsts.PodAnnotationQoSLevelDedicatedCores, + CPURequest: 1.0, + RampUp: false, + Annotations: map[string]string{ + apiconsts.PodAnnotationMemoryEnhancementNumaBinding: apiconsts.PodAnnotationMemoryEnhancementNumaBindingEnable, + }, + TopologyAwareAssignments: map[int]machine.CPUSet{ + 0: machine.NewCPUSet(0, 1, 2, 3), + }, + }, + }, + }, + regionInfo: types.RegionInfo{ + RegionName: "dedicated-numa-exclusive-xxx", + RegionType: types.QoSRegionTypeDedicatedNumaExclusive, + BindingNumas: machine.NewCPUSet(0), + }, + resourceEssentials: types.ResourceEssentials{ + EnableReclaim: true, + ResourceUpperBound: 90, + ResourceLowerBound: 4, + ReservedForAllocate: 0, + }, + controlEssentials: types.ControlEssentials{ + ControlKnobs: types.ControlKnob{ + types.ControlKnobNonReclaimedCPUSize: { + Value: 40, + Action: types.ControlKnobActionNone, + }, + }, + Indicators: types.Indicator{ + consts.MetricCPUCPIContainer: { + Current: 2.0, + Target: 1.0, + }, + consts.MetricMemBandwidthNuma: { + Current: 4, + Target: 40, + }, + }, + ReclaimOverlap: false, + }, + wantResult: types.ControlKnob{ + types.ControlKnobNonReclaimedCPUSize: { + Value: 4, + Action: types.ControlKnobActionNone, + }, + }, + }, } for _, tt := range tests { @@ -370,7 +444,7 @@ func TestPolicyCanonical(t *testing.T) { for podName, containerSet := range tt.containerInfo { podNames = append(podNames, podName) for containerName, info := range containerSet { - err = policy.metaReader.(*metacache.MetaCacheImp).AddContainer(podName, containerName, &info) + err = policy.metaReader.(*metacache.MetaCacheImp).AddContainer(podName, containerName, info) assert.Nil(t, err) } } diff --git a/pkg/agent/sysadvisor/plugin/qosaware/resource/cpu/region/region_dedicated_numa_exclusive.go b/pkg/agent/sysadvisor/plugin/qosaware/resource/cpu/region/region_dedicated_numa_exclusive.go index f5a851537..caad0937c 100644 --- a/pkg/agent/sysadvisor/plugin/qosaware/resource/cpu/region/region_dedicated_numa_exclusive.go +++ b/pkg/agent/sysadvisor/plugin/qosaware/resource/cpu/region/region_dedicated_numa_exclusive.go @@ -47,6 +47,8 @@ const ( type QoSRegionDedicatedNumaExclusive struct { *QoSRegionBase + + isNumaExclusive bool } // NewQoSRegionDedicatedNumaExclusive returns a region instance for dedicated cores @@ -68,6 +70,7 @@ func NewQoSRegionDedicatedNumaExclusive(ci *types.ContainerInfo, conf *config.Co } r.enableReclaim = r.EnableReclaim + r.isNumaExclusive = ci.IsNumaExclusive() return r } @@ -174,13 +177,27 @@ out: } func (r *QoSRegionDedicatedNumaExclusive) getControlKnobs() types.ControlKnob { - reclaimedCPUSize := 0 - if reclaimedInfo, ok := r.metaReader.GetPoolInfo(state.PoolNameReclaim); ok { - for _, numaID := range r.bindingNumas.ToSliceInt() { - reclaimedCPUSize += reclaimedInfo.TopologyAwareAssignments[numaID].Size() + var ( + cpuRequirement float64 + err error + ) + + if r.isNumaExclusive { + reclaimedCPUSize := 0 + if reclaimedInfo, ok := r.metaReader.GetPoolInfo(state.PoolNameReclaim); ok { + for _, numaID := range r.bindingNumas.ToSliceInt() { + reclaimedCPUSize += reclaimedInfo.TopologyAwareAssignments[numaID].Size() + } + } + cpuRequirement = r.ResourceUpperBound + r.ReservedForReclaim - float64(reclaimedCPUSize) + } else { + cpuRequirement, err = r.GetRegionRequest() + if err != nil { + // dedicated without numaExclusive region has a static estimation limited by min and max requirement + // just log and skip error here. + klog.Errorf("region %v GetRegionRequest fail: %v", r.name, err) } } - cpuRequirement := r.ResourceUpperBound + r.ReservedForReclaim - float64(reclaimedCPUSize) return types.ControlKnob{ types.ControlKnobNonReclaimedCPUSize: { @@ -217,3 +234,27 @@ func (r *QoSRegionDedicatedNumaExclusive) getPodCPICurrent() (float64, error) { return cpiSum / containerCnt, nil } + +func (r *QoSRegionDedicatedNumaExclusive) IsNumaExclusive() bool { + return r.isNumaExclusive +} + +func (r *QoSRegionDedicatedNumaExclusive) GetRegionRequest() (float64, error) { + var res float64 + + for podUID, containerSet := range r.podSet { + for containerName := range containerSet { + ci, ok := r.metaReader.GetContainerInfo(podUID, containerName) + if !ok || ci == nil { + err := fmt.Errorf("[qosaware-cpu] illegal container info of %v/%v", podUID, containerName) + return res, err + } + if ci.ContainerType == v1alpha1.ContainerType_MAIN { + res = ci.CPURequest + break + } + } + } + + return res, nil +} diff --git a/pkg/agent/sysadvisor/plugin/qosaware/server/cpu_server.go b/pkg/agent/sysadvisor/plugin/qosaware/server/cpu_server.go index f4f7634e4..2d10fecdc 100644 --- a/pkg/agent/sysadvisor/plugin/qosaware/server/cpu_server.go +++ b/pkg/agent/sysadvisor/plugin/qosaware/server/cpu_server.go @@ -376,8 +376,8 @@ func (cs *cpuServer) assemblePodEntries(calculationEntriesMap map[string]*cpuadv reclaimPoolCalculationResults, ok := getNumaCalculationResult(calculationEntriesMap, qrmstate.PoolNameReclaim, cpuadvisor.FakedContainerName, int64(numaID)) - if !ok { - // if no reclaimed pool exists, return the generated Block + if !ci.IsNumaExclusive() || !ok { + // if no reclaimed pool exists or container is not numaExclusive, return the generated Block block := NewBlock(uint64(cpuset.Size()), "") innerBlock := NewInnerBlock(block, int64(numaID), "", ci, numaCalculationResult) diff --git a/pkg/agent/sysadvisor/plugin/qosaware/server/cpu_server_test.go b/pkg/agent/sysadvisor/plugin/qosaware/server/cpu_server_test.go index e4cde0d26..005e96cfb 100644 --- a/pkg/agent/sysadvisor/plugin/qosaware/server/cpu_server_test.go +++ b/pkg/agent/sysadvisor/plugin/qosaware/server/cpu_server_test.go @@ -393,7 +393,8 @@ func TestCPUServerListAndWatch(t *testing.T) { PodUid: "pod1", ContainerName: "c1", Annotations: map[string]string{ - consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + consts.PodAnnotationMemoryEnhancementNumaExclusive: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, }, QosLevel: consts.PodAnnotationQoSLevelDedicatedCores, }, @@ -404,7 +405,7 @@ func TestCPUServerListAndWatch(t *testing.T) { UID: "pod1", Annotations: map[string]string{ consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, - consts.PodAnnotationMemoryEnhancementKey: "{\"numa_exclusive\":true}", + consts.PodAnnotationMemoryEnhancementKey: "{\"numa_exclusive\":true,\"numa_binding\":true}", }, }, Spec: v1.PodSpec{ @@ -524,7 +525,8 @@ func TestCPUServerListAndWatch(t *testing.T) { PodUid: "pod1", ContainerName: "c1", Annotations: map[string]string{ - consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + consts.PodAnnotationMemoryEnhancementNumaExclusive: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, }, QosLevel: consts.PodAnnotationQoSLevelDedicatedCores, }, @@ -535,7 +537,7 @@ func TestCPUServerListAndWatch(t *testing.T) { UID: "pod1", Annotations: map[string]string{ consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, - consts.PodAnnotationMemoryEnhancementKey: "{\"numa_exclusive\":true}", + consts.PodAnnotationMemoryEnhancementKey: "{\"numa_exclusive\":true,\"numa_binding\":true}", }, }, Spec: v1.PodSpec{ @@ -559,7 +561,8 @@ func TestCPUServerListAndWatch(t *testing.T) { PodUid: "pod1", ContainerName: "c2", Annotations: map[string]string{ - consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + consts.PodAnnotationMemoryEnhancementNumaExclusive: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, }, QosLevel: consts.PodAnnotationQoSLevelDedicatedCores, }, @@ -570,7 +573,7 @@ func TestCPUServerListAndWatch(t *testing.T) { UID: "pod1", Annotations: map[string]string{ consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, - consts.PodAnnotationMemoryEnhancementKey: "{\"numa_exclusive\":true}", + consts.PodAnnotationMemoryEnhancementKey: "{\"numa_exclusive\":true,\"numa_binding\":true}", }, }, Spec: v1.PodSpec{ @@ -767,7 +770,8 @@ func TestCPUServerListAndWatch(t *testing.T) { PodUid: "pod1", ContainerName: "c1", Annotations: map[string]string{ - consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + consts.PodAnnotationMemoryEnhancementNumaExclusive: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, }, QosLevel: consts.PodAnnotationQoSLevelDedicatedCores, }, @@ -778,7 +782,7 @@ func TestCPUServerListAndWatch(t *testing.T) { UID: "pod1", Annotations: map[string]string{ consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, - consts.PodAnnotationMemoryEnhancementKey: "{\"numa_exclusive\":true}", + consts.PodAnnotationMemoryEnhancementKey: "{\"numa_exclusive\":true,\"numa_binding\":true}", }, }, Spec: v1.PodSpec{ @@ -802,7 +806,8 @@ func TestCPUServerListAndWatch(t *testing.T) { PodUid: "pod1", ContainerName: "c2", Annotations: map[string]string{ - consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + consts.PodAnnotationMemoryEnhancementNumaExclusive: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, }, QosLevel: consts.PodAnnotationQoSLevelDedicatedCores, }, @@ -813,7 +818,7 @@ func TestCPUServerListAndWatch(t *testing.T) { UID: "pod1", Annotations: map[string]string{ consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, - consts.PodAnnotationMemoryEnhancementKey: "{\"numa_exclusive\":true}", + consts.PodAnnotationMemoryEnhancementKey: "{\"numa_exclusive\":true,\"numa_binding\":true}", }, }, Spec: v1.PodSpec{ @@ -837,7 +842,8 @@ func TestCPUServerListAndWatch(t *testing.T) { PodUid: "pod1", ContainerName: "c3", Annotations: map[string]string{ - consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + consts.PodAnnotationMemoryEnhancementNumaExclusive: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, }, QosLevel: consts.PodAnnotationQoSLevelDedicatedCores, }, @@ -848,7 +854,7 @@ func TestCPUServerListAndWatch(t *testing.T) { UID: "pod1", Annotations: map[string]string{ consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, - consts.PodAnnotationMemoryEnhancementKey: "{\"numa_exclusive\":true}", + consts.PodAnnotationMemoryEnhancementKey: "{\"numa_exclusive\":true,\"numa_binding\":true}", }, }, Spec: v1.PodSpec{