Skip to content

Commit

Permalink
feat: refactor config for cri plugin to relocate CRI property sandbox (…
Browse files Browse the repository at this point in the history 
…#5730)
  • Loading branch information
@bravebeaver
bravebeaver authored Feb 4, 2025
1 parent e32acd7 commit e396eea
Show file tree
Hide file tree
Showing 40 changed files with 5,483 additions and 17 deletions.
1 change: 1 addition & 0 deletions e2e/scenario_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1468,6 +1468,7 @@ func Test_Ubuntu2404Gen2(t *testing.T) {
runcVersions := getExpectedPackageVersions("runc", "ubuntu", "r2404")
ValidateContainerd2Properties(ctx, s, containerdVersions)
ValidateRunc12Properties(ctx, s, runcVersions)
ValidateContainerRuntimePlugins(ctx, s)
},
},
})
Expand Down
10 changes: 8 additions & 2 deletions e2e/validators.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -367,13 +367,19 @@ func ValidateContainerd2Properties(ctx context.Context, s *Scenario, versions []
require.Truef(s.T, strings.HasPrefix(versions[0], "2."), "expected moby-containerd version to start with '2.', got %v", versions[0])

ValidateInstalledPackageVersion(ctx, s, "moby-containerd", versions[0])
// assert that /etc/containerd/config.toml exists and does not contain deprecated properties from 1.7
ValidateFileExcludesContent(ctx, s, "/etc/containerd/config.toml", "CriuPath")
// assert that containerd.server service file does not contain LimitNOFILE
// https://github.com/containerd/containerd/blob/main/docs/containerd-2.0.md#limitnofile-configuration-has-been-removed
ValidateFileExcludesContent(ctx, s, "/etc/systemd/system/containerd.service", "LimitNOFILE")
}

func ValidateContainerRuntimePlugins(ctx context.Context, s *Scenario) {
// nri plugin is enabled by default
ValidateDirectoryContent(ctx, s, "/var/run/nri", []string{"nri.sock"})
// cri plugin has deprecated properties
// assert that /etc/containerd/config.toml exists and does not contain deprecated properties from 1.7
ValidateFileExcludesContent(ctx, s, "/etc/containerd/config.toml", "CriuPath")
// level=warning msg="Ignoring unknown key in TOML for plugin" error="strict mode: fields in the document are missing in the target struct" key=sandbox_image plugin=io.containerd.grpc.v1.cri
ValidateFileExcludesContent(ctx, s, "/etc/containerd/config.toml", "sandbox_image")
}

func ValidateRunc12Properties(ctx context.Context, s *Scenario, versions []string) {
Expand Down
271 changes: 257 additions & 14 deletions pkg/agent/baker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -697,14 +697,25 @@ func getContainerServiceFuncMap(config *datamodel.NodeBootstrappingConfiguration
return base64.StdEncoding.EncodeToString([]byte(kubenetCniTemplate))
},
"GetContainerdConfigContent": func() string {
output, err := containerdConfigFromTemplate(config, profile, containerdConfigTemplateString)
output, err := containerdConfigFromTemplate(config, profile, func(profile *datamodel.AgentPoolProfile) ContainerdConfigTemplate {
if profile.Is2404VHDDistro() {
return containerdV2ConfigTemplate
}
return containerdV1ConfigTemplate
}(profile))
if err != nil {
panic(err)
}
return output
},
"GetContainerdConfigNoGPUContent": func() string {
output, err := containerdConfigFromTemplate(config, profile, containerdConfigNoGpuTemplateString)
output, err := containerdConfigFromTemplate(config, profile, func(profile *datamodel.AgentPoolProfile) ContainerdConfigTemplate {
if profile.Is2404VHDDistro() {
return containerdV2NoGPUConfigTemplate
}
return containerdV1NoGPUConfigTemplate
}(profile))

if err != nil {
panic(err)
}
Expand Down Expand Up @@ -1245,7 +1256,17 @@ const kubenetCniTemplate = `
}
`

const containerdConfigTemplateString = `version = 2
type ContainerdConfigTemplate string

// this pains me, but to make it respect mutability of vmss tags,
// we cannot use go templates at runtime.
// CSE needs to be able to generate the full config, with all params,
// with the tags pulled from wireserver. this is a hack to avoid
// moving all the go templates to CSE -- we allow two options,
// duplicate them in CSE base64-encoded, and pick the right one.
// they're identical except for GPU runtime class.
const (
containerdV1ConfigTemplate ContainerdConfigTemplate = `version = 2
oom_score = 0{{if HasDataDir }}
root = "{{GetDataDir}}"{{- end}}
[plugins."io.containerd.grpc.v1.cri"]
Expand Down Expand Up @@ -1366,15 +1387,236 @@ root = "{{GetDataDir}}"{{- end}}
ConfigPath = "/opt/confidential-containers/share/defaults/kata-containers/configuration-clh-snp.toml"
{{- end}}
`

// this pains me, but to make it respect mutability of vmss tags,
// we cannot use go templates at runtime.
// CSE needs to be able to generate the full config, with all params,
// with the tags pulled from wireserver. this is a hack to avoid
// moving all the go templates to CSE -- we allow two options,
// duplicate them in CSE base64-encoded, and pick the right one.
// they're identical except for GPU runtime class.
const containerdConfigNoGpuTemplateString = `version = 2
containerdV2ConfigTemplate ContainerdConfigTemplate = `version = 2
oom_score = 0{{if HasDataDir }}
root = "{{GetDataDir}}"{{- end}}
[plugins."io.containerd.grpc.v1.cri"]
[plugins.'io.containerd.cri.v1.images'.pinned_images]
sandbox = "{{GetPodInfraContainerSpec}}"
[plugins."io.containerd.grpc.v1.cri".containerd]
{{- if TeleportEnabled }}
snapshotter = "teleportd"
disable_snapshot_annotations = false
{{- else}}
{{- if IsKata }}
disable_snapshot_annotations = false
{{- end}}
{{- end}}
{{- if IsArtifactStreamingEnabled }}
snapshotter = "overlaybd"
disable_snapshot_annotations = false
{{- end}}
{{- if IsNSeriesSKU }}
default_runtime_name = "nvidia-container-runtime"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-container-runtime]
runtime_type = "io.containerd.runc.v2"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-container-runtime.options]
BinaryName = "/usr/bin/nvidia-container-runtime"
{{- if IsCgroupV2 }}
SystemdCgroup = true
{{- end}}
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.untrusted]
runtime_type = "io.containerd.runc.v2"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.untrusted.options]
BinaryName = "/usr/bin/nvidia-container-runtime"
{{- else}}
default_runtime_name = "runc"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
runtime_type = "io.containerd.runc.v2"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
BinaryName = "/usr/bin/runc"
{{- if IsCgroupV2 }}
SystemdCgroup = true
{{- end}}
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.untrusted]
runtime_type = "io.containerd.runc.v2"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.untrusted.options]
BinaryName = "/usr/bin/runc"
{{- end}}
{{- if IsKrustlet }}
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.spin]
runtime_type = "io.containerd.spin.v2"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.slight]
runtime_type = "io.containerd.slight-v0-3-0.v1"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.spin-v0-3-0]
runtime_type = "io.containerd.spin-v0-3-0.v1"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.slight-v0-3-0]
runtime_type = "io.containerd.slight-v0-3-0.v1"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.spin-v0-5-1]
runtime_type = "io.containerd.spin-v0-5-1.v1"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.slight-v0-5-1]
runtime_type = "io.containerd.slight-v0-5-1.v1"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.spin-v0-8-0]
runtime_type = "io.containerd.spin-v0-8-0.v1"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.slight-v0-8-0]
runtime_type = "io.containerd.slight-v0-8-0.v1"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.wws-v0-8-0]
runtime_type = "io.containerd.wws-v0-8-0.v1"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.spin-v0-15-1]
runtime_type = "io.containerd.spin.v2"
{{- end}}
{{- if and (IsKubenet) (not HasCalicoNetworkPolicy) }}
[plugins."io.containerd.grpc.v1.cri".cni]
bin_dir = "/opt/cni/bin"
conf_dir = "/etc/cni/net.d"
conf_template = "/etc/containerd/kubenet_template.conf"
{{- end}}
{{- if IsKubernetesVersionGe "1.22.0"}}
[plugins."io.containerd.grpc.v1.cri".registry]
config_path = "/etc/containerd/certs.d"
{{- end}}
[plugins."io.containerd.grpc.v1.cri".registry.headers]
X-Meta-Source-Client = ["azure/aks"]
[metrics]
address = "0.0.0.0:10257"
{{- if TeleportEnabled }}
[proxy_plugins]
[proxy_plugins.teleportd]
type = "snapshot"
address = "/run/teleportd/snapshotter.sock"
{{- end}}
{{- if IsArtifactStreamingEnabled }}
[proxy_plugins]
[proxy_plugins.overlaybd]
type = "snapshot"
address = "/run/overlaybd-snapshotter/overlaybd.sock"
{{- end}}
{{- if IsKata }}
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata]
runtime_type = "io.containerd.kata.v2"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.katacli]
runtime_type = "io.containerd.runc.v1"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.katacli.options]
NoPivotRoot = false
NoNewKeyring = false
ShimCgroup = ""
IoUid = 0
IoGid = 0
BinaryName = "/usr/bin/kata-runtime"
Root = ""
CriuPath = ""
SystemdCgroup = false
[proxy_plugins]
[proxy_plugins.tardev]
type = "snapshot"
address = "/run/containerd/tardev-snapshotter.sock"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-cc]
snapshotter = "tardev"
runtime_type = "io.containerd.kata-cc.v2"
privileged_without_host_devices = true
pod_annotations = ["io.katacontainers.*"]
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-cc.options]
ConfigPath = "/opt/confidential-containers/share/defaults/kata-containers/configuration-clh-snp.toml"
{{- end}}
`
containerdV2NoGPUConfigTemplate ContainerdConfigTemplate = `version = 2
oom_score = 0{{if HasDataDir }}
root = "{{GetDataDir}}"{{- end}}
[plugins."io.containerd.grpc.v1.cri"]
[plugins.'io.containerd.cri.v1.images'.pinned_images]
sandbox = "{{GetPodInfraContainerSpec}}"
[plugins."io.containerd.grpc.v1.cri".containerd]
{{- if TeleportEnabled }}
snapshotter = "teleportd"
disable_snapshot_annotations = false
{{- else}}
{{- if IsKata }}
disable_snapshot_annotations = false
{{- end}}
{{- end}}
{{- if IsArtifactStreamingEnabled }}
snapshotter = "overlaybd"
disable_snapshot_annotations = false
{{- end}}
default_runtime_name = "runc"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
runtime_type = "io.containerd.runc.v2"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
BinaryName = "/usr/bin/runc"
{{- if IsCgroupV2 }}
SystemdCgroup = true
{{- end}}
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.untrusted]
runtime_type = "io.containerd.runc.v2"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.untrusted.options]
BinaryName = "/usr/bin/runc"
{{- if IsKrustlet }}
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.spin]
runtime_type = "io.containerd.spin.v2"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.slight]
runtime_type = "io.containerd.slight-v0-3-0.v1"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.spin-v0-3-0]
runtime_type = "io.containerd.spin-v0-3-0.v1"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.slight-v0-3-0]
runtime_type = "io.containerd.slight-v0-3-0.v1"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.spin-v0-5-1]
runtime_type = "io.containerd.spin-v0-5-1.v1"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.slight-v0-5-1]
runtime_type = "io.containerd.slight-v0-5-1.v1"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.spin-v0-8-0]
runtime_type = "io.containerd.spin-v0-8-0.v1"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.slight-v0-8-0]
runtime_type = "io.containerd.slight-v0-8-0.v1"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.wws-v0-8-0]
runtime_type = "io.containerd.wws-v0-8-0.v1"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.spin-v0-15-1]
runtime_type = "io.containerd.spin.v2"
{{- end}}
{{- if and (IsKubenet) (not HasCalicoNetworkPolicy) }}
[plugins."io.containerd.grpc.v1.cri".cni]
bin_dir = "/opt/cni/bin"
conf_dir = "/etc/cni/net.d"
conf_template = "/etc/containerd/kubenet_template.conf"
{{- end}}
{{- if IsKubernetesVersionGe "1.22.0"}}
[plugins."io.containerd.grpc.v1.cri".registry]
config_path = "/etc/containerd/certs.d"
{{- end}}
[plugins."io.containerd.grpc.v1.cri".registry.headers]
X-Meta-Source-Client = ["azure/aks"]
[metrics]
address = "0.0.0.0:10257"
{{- if TeleportEnabled }}
[proxy_plugins]
[proxy_plugins.teleportd]
type = "snapshot"
address = "/run/teleportd/snapshotter.sock"
{{- end}}
{{- if IsArtifactStreamingEnabled }}
[proxy_plugins]
[proxy_plugins.overlaybd]
type = "snapshot"
address = "/run/overlaybd-snapshotter/overlaybd.sock"
{{- end}}
{{- if IsKata }}
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata]
runtime_type = "io.containerd.kata.v2"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.katacli]
runtime_type = "io.containerd.runc.v1"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.katacli.options]
NoPivotRoot = false
NoNewKeyring = false
ShimCgroup = ""
IoUid = 0
IoGid = 0
BinaryName = "/usr/bin/kata-runtime"
Root = ""
CriuPath = ""
SystemdCgroup = false
[proxy_plugins]
[proxy_plugins.tardev]
type = "snapshot"
address = "/run/containerd/tardev-snapshotter.sock"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-cc]
snapshotter = "tardev"
runtime_type = "io.containerd.kata-cc.v2"
privileged_without_host_devices = true
pod_annotations = ["io.katacontainers.*"]
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-cc.options]
ConfigPath = "/opt/confidential-containers/share/defaults/kata-containers/configuration-clh-snp.toml"
{{- end}}
`
containerdV1NoGPUConfigTemplate ContainerdConfigTemplate = `version = 2
oom_score = 0{{if HasDataDir }}
root = "{{GetDataDir}}"{{- end}}
[plugins."io.containerd.grpc.v1.cri"]
Expand Down Expand Up @@ -1480,16 +1722,17 @@ root = "{{GetDataDir}}"{{- end}}
ConfigPath = "/opt/confidential-containers/share/defaults/kata-containers/configuration-clh-snp.toml"
{{- end}}
`
)

func containerdConfigFromTemplate(
config *datamodel.NodeBootstrappingConfiguration,
profile *datamodel.AgentPoolProfile,
tmpl string,
tmpl ContainerdConfigTemplate,
) (string, error) {
parameters := getParameters(config)
variables := getCustomDataVariables(config)
bakerFuncMap := getBakerFuncMap(config, parameters, variables)
containerdConfigTemplate := template.Must(template.New("kubenet").Funcs(bakerFuncMap).Parse(tmpl))
containerdConfigTemplate := template.Must(template.New("kubenet").Funcs(bakerFuncMap).Parse(string(tmpl)))
var b bytes.Buffer
if err := containerdConfigTemplate.Execute(&b, profile); err != nil {
return "", fmt.Errorf("failed to execute sysctl template: %w", err)
Expand Down
22 changes: 22 additions & 0 deletions pkg/agent/baker_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1645,6 +1645,28 @@ oom_score = 0
Expect(containerdUlimitContent).NotTo(ContainSubstring("LimitNOFILE=1048"))
Expect(containerdUlimitContent).To(ContainSubstring("LimitMEMLOCK=75000"))
}),
Entry("AKSUbuntu2404 containerd v2 CRI plugin config should not have deprecated features", "AKSUbuntu2404", ">=1.32.x",
func(config *datamodel.NodeBootstrappingConfiguration) {
config.ContainerService.Properties.AgentPoolProfiles[0].KubernetesConfig = &datamodel.KubernetesConfig{
ContainerRuntime: datamodel.Containerd,
}
config.ContainerService.Properties.AgentPoolProfiles[0].Distro = datamodel.AKSUbuntuContainerd2404
config.ContainerService.Properties.OrchestratorProfile.OrchestratorVersion = "1.32.0"
}, func(o *nodeBootstrappingOutput) {
containerdConfigFileContent, err := getBase64DecodedValue([]byte(o.vars["CONTAINERD_CONFIG_CONTENT"]))
Expect(err).To(BeNil())
expectedContainerdV2CriConfig := `
[plugins."io.containerd.grpc.v1.cri"]
[plugins.'io.containerd.cri.v1.images'.pinned_images]
sandbox = ""
`
deprecatedContainerdV1CriConfig := `
[plugins."io.containerd.grpc.v1.cri"]
sandbox_image = ""
`
Expect(containerdConfigFileContent).To(ContainSubstring(expectedContainerdV2CriConfig))
Expect(containerdConfigFileContent).NotTo(ContainSubstring(deprecatedContainerdV1CriConfig))
}),
)
})

Expand Down
Loading

0 comments on commit e396eea

Please sign in to comment.