Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

runtime: Allocate default workload vcpus #282

Merged
merged 1 commit into from
Jan 9, 2025
Merged

runtime: Allocate default workload vcpus #282

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions src/runtime/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,7 @@ STRATOVIRTPATH = $(STRATOVIRTBINDIR)/$(STRATOVIRTCMD)
STRATOVIRTVALIDHYPERVISORPATHS := [\"$(STRATOVIRTPATH)\"]

# Default number of vCPUs
DEFVCPUS := 1
DEFVCPUS ?= 1
# Default maximum number of vCPUs
DEFMAXVCPUS := 0
# Default memory size in MiB
Expand Down Expand Up @@ -263,8 +263,9 @@ DEFSANDBOXCGROUPONLY ?= false
DEFSTATICRESOURCEMGMT ?= false
DEFSTATICRESOURCEMGMT_TEE = true

# Default memory for use for workloads within the sandbox if no specific workload memory value is requested
# Default memory and vcpus for use for workloads within the sandbox if no specific workload values are requested
DEFSTATICSANDBOXWORKLOADMEM ?= 2048
DEFSTATICSANDBOXWORKLOADVCPUS ?= 1

DEFBINDMOUNTS := []

Expand Down Expand Up @@ -694,6 +695,7 @@ USER_VARS += DEFSTATICRESOURCEMGMT_FC
USER_VARS += DEFSTATICRESOURCEMGMT_STRATOVIRT
USER_VARS += DEFSTATICRESOURCEMGMT_TEE
USER_VARS += DEFSTATICSANDBOXWORKLOADMEM
USER_VARS += DEFSTATICSANDBOXWORKLOADVCPUS
USER_VARS += DEFBINDMOUNTS
USER_VARS += DEFSERVICEOFFLOAD
USER_VARS += DEFVFIOMODE
Expand Down
7 changes: 6 additions & 1 deletion src/runtime/config/configuration-clh-snp.toml.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ kernel_params = "@KERNELPARAMS@"
# < 0 --> will be set to the actual number of physical cores
# > 0 <= number of physical cores --> will be set to the specified number
# > number of physical cores --> will be set to the actual number of physical cores
default_vcpus = 1
default_vcpus = @DEFVCPUS@

# Default maximum number of vCPUs per SB/VM:
# unspecified or == 0 --> will be set to the actual number of physical cores or to the maximum number
Expand Down Expand Up @@ -398,6 +398,11 @@ static_sandbox_resource_mgmt=@DEFSTATICRESOURCEMGMT_TEE@
# default amount of memory available within the sandbox.
static_sandbox_default_workload_mem=@DEFSTATICSANDBOXWORKLOADMEM@

# If set, the runtime will use the value as the default number of vcpus for the sandbox when no workload vcpu request is passed
# down to the shim via the OCI when static sandbox resource management is enabled. With this, we ensure that workloads have a proper
# default amount of vcpus available within the sandbox.
static_sandbox_default_workload_vcpus=@DEFSTATICSANDBOXWORKLOADVCPUS@

# If specified, sandbox_bind_mounts identifieds host paths to be mounted (ro) into the sandboxes shared path.
# This is only valid if filesystem sharing is utilized. The provided path(s) will be bindmounted into the shared fs directory.
# If defaults are utilized, these mounts should be available in the guest at `/run/kata-containers/shared/containers/sandbox-mounts`
Expand Down
7 changes: 6 additions & 1 deletion src/runtime/config/configuration-clh.toml.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ kernel_params = "@KERNELPARAMS@"
# < 0 --> will be set to the actual number of physical cores
# > 0 <= number of physical cores --> will be set to the specified number
# > number of physical cores --> will be set to the actual number of physical cores
default_vcpus = 1
default_vcpus = @DEFVCPUS@

# Default maximum number of vCPUs per SB/VM:
# unspecified or == 0 --> will be set to the actual number of physical cores or to the maximum number
Expand Down Expand Up @@ -418,6 +418,11 @@ static_sandbox_resource_mgmt=@DEFSTATICRESOURCEMGMT_CLH@
# default amount of memory available within the sandbox.
static_sandbox_default_workload_mem=@DEFSTATICSANDBOXWORKLOADMEM@

# If set, the runtime will use the value as the default number of vcpus for the sandbox when no workload vcpu request is passed
# down to the shim via the OCI when static sandbox resource management is enabled. With this, we ensure that workloads have a proper
# default amount of vcpus available within the sandbox.
static_sandbox_default_workload_vcpus=@DEFSTATICSANDBOXWORKLOADVCPUS@

# If specified, sandbox_bind_mounts identifieds host paths to be mounted (ro) into the sandboxes shared path.
# This is only valid if filesystem sharing is utilized. The provided path(s) will be bindmounted into the shared fs directory.
# If defaults are utilized, these mounts should be available in the guest at `/run/kata-containers/shared/containers/sandbox-mounts`
Expand Down
2 changes: 1 addition & 1 deletion src/runtime/pkg/katautils/config-settings.go.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ var systemdUnitName = "kata-containers.target"
const defaultKernelParams = ""
const defaultMachineType = "q35"

const defaultVCPUCount uint32 = 1
const defaultVCPUCount uint32 = 0
const defaultMaxVCPUCount uint32 = 0
const defaultMemSize uint32 = 2048 // MiB
const defaultMemSlots uint32 = 10
Expand Down
38 changes: 20 additions & 18 deletions src/runtime/pkg/katautils/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,24 +170,25 @@ type hypervisor struct {
}

type runtime struct {
InterNetworkModel string `toml:"internetworking_model"`
JaegerEndpoint string `toml:"jaeger_endpoint"`
JaegerUser string `toml:"jaeger_user"`
JaegerPassword string `toml:"jaeger_password"`
VfioMode string `toml:"vfio_mode"`
GuestSeLinuxLabel string `toml:"guest_selinux_label"`
SandboxBindMounts []string `toml:"sandbox_bind_mounts"`
Experimental []string `toml:"experimental"`
Tracing bool `toml:"enable_tracing"`
DisableNewNetNs bool `toml:"disable_new_netns"`
DisableGuestSeccomp bool `toml:"disable_guest_seccomp"`
EnableVCPUsPinning bool `toml:"enable_vcpus_pinning"`
Debug bool `toml:"enable_debug"`
SandboxCgroupOnly bool `toml:"sandbox_cgroup_only"`
StaticSandboxResourceMgmt bool `toml:"static_sandbox_resource_mgmt"`
EnablePprof bool `toml:"enable_pprof"`
DisableGuestEmptyDir bool `toml:"disable_guest_empty_dir"`
StaticSandboxWorkloadDefaultMem uint32 `toml:"static_sandbox_default_workload_mem"`
InterNetworkModel string `toml:"internetworking_model"`
JaegerEndpoint string `toml:"jaeger_endpoint"`
JaegerUser string `toml:"jaeger_user"`
JaegerPassword string `toml:"jaeger_password"`
VfioMode string `toml:"vfio_mode"`
GuestSeLinuxLabel string `toml:"guest_selinux_label"`
SandboxBindMounts []string `toml:"sandbox_bind_mounts"`
Experimental []string `toml:"experimental"`
Tracing bool `toml:"enable_tracing"`
DisableNewNetNs bool `toml:"disable_new_netns"`
DisableGuestSeccomp bool `toml:"disable_guest_seccomp"`
EnableVCPUsPinning bool `toml:"enable_vcpus_pinning"`
Debug bool `toml:"enable_debug"`
SandboxCgroupOnly bool `toml:"sandbox_cgroup_only"`
StaticSandboxResourceMgmt bool `toml:"static_sandbox_resource_mgmt"`
EnablePprof bool `toml:"enable_pprof"`
DisableGuestEmptyDir bool `toml:"disable_guest_empty_dir"`
StaticSandboxWorkloadDefaultMem uint32 `toml:"static_sandbox_default_workload_mem"`
StaticSandboxWorkloadDefaultVcpus float32 `toml:"static_sandbox_default_workload_vcpus"`
}

type agent struct {
Expand Down Expand Up @@ -1588,6 +1589,7 @@ func LoadConfiguration(configPath string, ignoreLogging bool) (resolvedConfigPat
config.GuestSeLinuxLabel = tomlConf.Runtime.GuestSeLinuxLabel
config.StaticSandboxResourceMgmt = tomlConf.Runtime.StaticSandboxResourceMgmt
config.StaticSandboxWorkloadDefaultMem = tomlConf.Runtime.StaticSandboxWorkloadDefaultMem
config.StaticSandboxWorkloadDefaultVcpus = tomlConf.Runtime.StaticSandboxWorkloadDefaultVcpus
config.SandboxCgroupOnly = tomlConf.Runtime.SandboxCgroupOnly
config.DisableNewNetNs = tomlConf.Runtime.DisableNewNetNs
config.EnablePprof = tomlConf.Runtime.EnablePprof
Expand Down
12 changes: 10 additions & 2 deletions src/runtime/pkg/oci/utils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,9 @@ type RuntimeConfig struct {
// Memory to allocate for workloads within the sandbox when workload memory is unspecified
StaticSandboxWorkloadDefaultMem uint32

// vcpus to allocate for workloads within the sandbox when workload vcpus is unspecified
StaticSandboxWorkloadDefaultVcpus float32

// Determines if create a netns for hypervisor process
DisableNewNetNs bool

Expand Down Expand Up @@ -996,6 +999,8 @@ func SandboxConfig(ocispec specs.Spec, runtime RuntimeConfig, bundlePath, cid st

StaticWorkloadDefaultMem: runtime.StaticSandboxWorkloadDefaultMem,

StaticWorkloadDefaultVcpus: runtime.StaticSandboxWorkloadDefaultVcpus,

ShmSize: shmSize,

VfioMode: runtime.VfioMode,
Expand All @@ -1022,11 +1027,14 @@ func SandboxConfig(ocispec specs.Spec, runtime RuntimeConfig, bundlePath, cid st
// with the base number of CPU/memory (which is equal to the default CPU/memory specified for the runtime
// configuration or annotations) as well as any specified workload resources.
if sandboxConfig.StaticResourceMgmt {
// If no Limits are set in pod config, use StaticWorkloadDefaultMem to ensure the containers generally
// have a reasonable amount of memory available
// If no Limits are set in pod config, use StaticWorkloadDefaultMem/Vcpus to ensure the containers generally
// have a reasonable amount of resources available
if sandboxConfig.SandboxResources.WorkloadMemMB == 0 {
sandboxConfig.SandboxResources.WorkloadMemMB = sandboxConfig.StaticWorkloadDefaultMem
}
if sandboxConfig.SandboxResources.WorkloadCPUs == 0 {
sandboxConfig.SandboxResources.WorkloadCPUs = sandboxConfig.StaticWorkloadDefaultVcpus
}

sandboxConfig.SandboxResources.BaseCPUs = sandboxConfig.HypervisorConfig.NumVCPUsF
sandboxConfig.SandboxResources.BaseMemMB = sandboxConfig.HypervisorConfig.MemorySize
Expand Down
2 changes: 1 addition & 1 deletion src/runtime/virtcontainers/hypervisor.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ const (

procCPUInfo = "/proc/cpuinfo"

defaultVCPUs = float32(1)
defaultVCPUs = float32(0)
ms-mahuber marked this conversation as resolved.
Show resolved Hide resolved
// 2 GiB
defaultMemSzMiB = 2048

Expand Down
2 changes: 2 additions & 0 deletions src/runtime/virtcontainers/sandbox.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,8 @@ type SandboxConfig struct {

StaticWorkloadDefaultMem uint32

StaticWorkloadDefaultVcpus float32

// Memory to allocate for workloads within the sandbox when workload memory is unspecified
ShmSize uint64

Expand Down
2 changes: 1 addition & 1 deletion tools/osbuilder/node-builder/azure-linux/package_build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ source "${common_file}"

# these options ensure we produce the proper CLH config file
runtime_make_flags="SKIP_GO_VERSION_CHECK=1 QEMUCMD= FCCMD= ACRNCMD= STRATOVIRTCMD= DEFAULT_HYPERVISOR=cloud-hypervisor
DEFMEMSZ=0 DEFSTATICSANDBOXWORKLOADMEM=512 DEFVIRTIOFSDAEMON=${VIRTIOFSD_BINARY_LOCATION} PREFIX=${INSTALL_PATH_PREFIX}"
DEFMEMSZ=0 DEFSTATICSANDBOXWORKLOADMEM=512 DEFVCPUS=0 DEFSTATICSANDBOXWORKLOADVCPUS=1 DEFVIRTIOFSDAEMON=${VIRTIOFSD_BINARY_LOCATION} PREFIX=${INSTALL_PATH_PREFIX}"

# - for vanilla Kata we use the kernel binary. For ConfPods we use IGVM, so no need to provide kernel path.
# - for vanilla Kata we explicitly set DEFSTATICRESOURCEMGMT_CLH. For ConfPods,
Expand Down
Loading