diff --git a/README.md b/README.md index 86eced64..63200a3b 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ This document details configuring the Virtual Kubelet ACI provider. Virtual Kubelet's ACI provider relies heavily on the feature set that Azure Container Instances provide. Please check the Azure documentation accurate details on region availability, pricing and new features. The list here attempts to give an accurate reference for the features we support in ACI and the ACI provider within Virtual Kubelet. -### features +### Features * Volumes: empty dir, github repo, projection, Azure Files, Azure Files CSI drivers * Secure env variables, config maps @@ -36,6 +36,7 @@ Virtual Kubelet's ACI provider relies heavily on the feature set that Azure Cont * Basic Azure Networking support within AKS virtual node * [Exec support](https://docs.microsoft.com/azure/container-instances/container-instances-exec) for container instances * Azure Monitor integration or formally known as OMS +* Support for init-containers ([use init containers](#Create-pod-with-init-containers)) ### Limitations @@ -44,7 +45,6 @@ Virtual Kubelet's ACI provider relies heavily on the feature set that Azure Cont * [Limitations](https://docs.microsoft.com/azure/container-instances/container-instances-vnet) with VNet * VNet peering * Argument support for exec -* Init containers * [Host aliases](https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/) support * downward APIs (i.e podIP) @@ -650,6 +650,28 @@ Output: ``` --> +### Create pod with init containers +Multiple init containers can be specified in the podspec similar to how containers are specified + +```yaml +spec: + initContainers: + - image: + name: init-container-01 + command: [ "/bin/sh" ] + args: [ "-c", "echo \"Hi\"" ] + - image: + name: init-container-02 + command: [ "/bin/sh" ] + args: [ "-c", "echo \"Hi\"" ] + containers: + - image: + imagePullPolicy: Always + name: container + command: [ "/bin/sh" ] +``` +More information on init containers can be found in [Kubernetes](https://kubernetes.io/docs/concepts/workloads/pods/init-containers/) and [ACI](https://docs.microsoft.com/en-us/azure/container-instances/container-instances-init-container) documentations + ## Work around for the virtual kubelet pod If your pod that's scheduled onto the Virtual Kubelet node is in a pending state please add this workaround to your Virtual Kubelet pod spec. diff --git a/e2e/fixtures/initcontainers_ordertest_pod.yml b/e2e/fixtures/initcontainers_ordertest_pod.yml new file mode 100644 index 00000000..1f3c554c --- /dev/null +++ b/e2e/fixtures/initcontainers_ordertest_pod.yml @@ -0,0 +1,51 @@ +apiVersion: v1 +kind: Pod +metadata: + name: vk-e2e-initcontainers-order + namespace: vk-test +spec: + initContainers: + - image: alpine + name: init-container-01 + command: [ "/bin/sh" ] + args: [ "-c", "echo Hi from init-container-01 >> /mnt/azure/newfile.txt" ] + volumeMounts: + - name: azure + mountPath: /mnt/azure + - image: alpine + name: init-container-02 + command: [ "/bin/sh" ] + args: [ "-c", "echo Hi from init-container-02 >> /mnt/azure/newfile.txt" ] + volumeMounts: + - name: azure + mountPath: /mnt/azure + containers: + - image: alpine + imagePullPolicy: Always + name: container + command: [ + "sh", + "-c", + "echo Hi from container >> /mnt/azure/newfile.txt; while sleep 10; do cat /mnt/azure/newfile.txt; done;" + ] + resources: + requests: + memory: 1G + cpu: 1 + volumeMounts: + - name: azure + mountPath: /mnt/azure + nodeSelector: + kubernetes.io/role: agent + beta.kubernetes.io/os: linux + type: virtual-kubelet + tolerations: + - key: virtual-kubelet.io/provider + operator: Exists + volumes: + - name: azure + csi: + driver: file.csi.azure.com + volumeAttributes: + secretName: csidriversecret # required + shareName: vncsidriversharename # required diff --git a/e2e/initcontainer_test.go b/e2e/initcontainer_test.go new file mode 100644 index 00000000..f9651705 --- /dev/null +++ b/e2e/initcontainer_test.go @@ -0,0 +1,113 @@ + +package e2e + +import ( + "testing" + "time" + "io/ioutil" + "os/exec" + "os" + + "gotest.tools/assert" +) + +func TestPodWithInitContainersOrder(t *testing.T) { + // delete the namespace first + cmd := kubectl("delete", "namespace", "vk-test", "--ignore-not-found") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatal(string(out)) + } + + // create namespace + cmd = kubectl("apply", "-f", "fixtures/namespace.yml") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatal(string(out)) + } + + testStorageAccount := os.Getenv("CSI_DRIVER_STORAGE_ACCOUNT_NAME") + testStorageKey := os.Getenv("CSI_DRIVER_STORAGE_ACCOUNT_KEY") + + cmd = kubectl("create", "secret", "generic", "csidriversecret", "--from-literal", "azurestorageaccountname="+testStorageAccount, "--from-literal", "azurestorageaccountkey="+testStorageKey, "--namespace=vk-test") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatal(string(out)) + } + + cmd = kubectl("apply", "-f", "fixtures/initcontainers_ordertest_pod.yml") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatal(string(out)) + } + deadline, ok := t.Deadline() + timeout := time.Until(deadline) + if !ok { + timeout = 300 * time.Second + } + cmd = kubectl("wait", "--for=condition=ready", "--timeout="+timeout.String(), "pod/vk-e2e-initcontainers-order", "--namespace=vk-test") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatal(string(out)) + } + t.Log("success create pod") + + // query metrics + deadline = time.Now().Add(10 * time.Minute) + for { + t.Log("query metrics ....") + cmd = kubectl("get", "--raw", "/apis/metrics.k8s.io/v1beta1/namespaces/vk-test/pods/vk-e2e-initcontainers-order") + out, err := cmd.CombinedOutput() + if time.Now().After(deadline) { + t.Fatal("failed to query pod's stats from metrics server API") + } + if err == nil { + t.Logf("success query metrics %s", string(out)) + break + } + time.Sleep(10 * time.Second) + } + + // download file created by pod + cmd = exec.Command("az", "storage", "file", "download", "--account-name", testStorageAccount, "--account-key", testStorageKey, "-s", "vncsidriversharename", "-p", "newfile.txt") + cmd.Env = os.Environ() + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatal(string(out)) + } + t.Log("file newfile.txt downloaded from storage account") + + file, err := ioutil.ReadFile("newfile.txt") + if err != nil { + t.Fatal("could not read downloaded file") + } + t.Log("read file content successfully") + + fileContent := string(file) + expectedString := "Hi from init-container-01\nHi from init-container-02\nHi from container\n" + assert.Equal(t, fileContent, expectedString, "file content doesn't match expected value") + + // check pod status + t.Log("get pod status ....") + cmd = kubectl("get", "pod", "--field-selector=status.phase=Running", "--namespace=vk-test", "--output=jsonpath={.items..metadata.name}") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatal(string(out)) + } + if string(out) != "vk-e2e-initcontainers-order" { + t.Fatal("failed to get pod's status") + } + t.Logf("success query pod status %s", string(out)) + + // check container status + t.Log("get container status ....") + cmd = kubectl("get", "pod", "vk-e2e-initcontainers-order", "--namespace=vk-test", "--output=jsonpath={.status.containerStatuses[0].ready}") + out, err = cmd.CombinedOutput() + if err != nil { + t.Fatal(string(out)) + } + if string(out) != "true" { + t.Fatal("failed to get pod's status") + } + t.Logf("success query container status %s", string(out)) + + t.Log("clean up pod") + cmd = kubectl("delete", "namespace", "vk-test", "--ignore-not-found") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatal(string(out)) + } +} diff --git a/e2e/pods_test.go b/e2e/pods_test.go index 70c22263..94f03e6a 100644 --- a/e2e/pods_test.go +++ b/e2e/pods_test.go @@ -18,6 +18,7 @@ func TestPodLifecycle(t *testing.T) { if out, err := cmd.CombinedOutput(); err != nil { t.Fatal(string(out)) } + cmd = kubectl("apply", "-f", "fixtures/hpa.yml", "--namespace=vk-test") if out, err := cmd.CombinedOutput(); err != nil { t.Fatal(string(out)) @@ -99,9 +100,10 @@ func TestPodLifecycle(t *testing.T) { } t.Log("success query exec on the container") - t.Log("clean up pod") + t.Log("clean up") cmd = kubectl("delete", "namespace", "vk-test", "--ignore-not-found") if out, err := cmd.CombinedOutput(); err != nil { t.Fatal(string(out)) } } + diff --git a/hack/e2e/aks.sh b/hack/e2e/aks.sh index 740e0982..5dfa97c2 100755 --- a/hack/e2e/aks.sh +++ b/hack/e2e/aks.sh @@ -20,7 +20,7 @@ if [ "$PR_RAND" = "" ]; then fi : "${RESOURCE_GROUP:=vk-aci-test-$RANDOM_NUM}" -: "${LOCATION:=westus2}" +: "${LOCATION:=eastus2}" : "${CLUSTER_NAME:=${RESOURCE_GROUP}}" : "${NODE_COUNT:=1}" : "${CHART_NAME:=vk-aci-test-aks}" diff --git a/pkg/provider/aci.go b/pkg/provider/aci.go index 6450f157..17d1e4de 100644 --- a/pkg/provider/aci.go +++ b/pkg/provider/aci.go @@ -303,7 +303,15 @@ func (p *ACIProvider) CreatePod(ctx context.Context, pod *v1.Pod) error { return err } + + // get initContainers + initContainers, err := p.getInitContainers(ctx, pod) + if err != nil { + return err + } + // assign all the things + cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.InitContainers = &initContainers cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Containers = containers cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Volumes = &volumes cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.ImageRegistryCredentials = creds @@ -840,6 +848,91 @@ func readDockerConfigJSONSecret(secret *v1.Secret, ips []azaci.ImageRegistryCred return ips, err } +//verify if Container is properly declared for the use on ACI +func (p *ACIProvider) verifyContainer(container *v1.Container) error { + if len(container.Command) == 0 && len(container.Args) > 0 { + return errdefs.InvalidInput("ACI does not support providing args without specifying the command. Please supply both command and args to the pod spec.") + } + return nil +} + +//this method is used for both initConainers and containers +func (p *ACIProvider) getCommand(container *v1.Container) *[]string { + command := append(container.Command, container.Args...) + return &command +} + +//get VolumeMounts declared on Container as []aci.VolumeMount +func (p *ACIProvider) getVolumeMounts(container *v1.Container) *[]azaci.VolumeMount { + volumeMounts := make([]azaci.VolumeMount, 0, len(container.VolumeMounts)) + for i := range container.VolumeMounts { + volumeMounts = append(volumeMounts, azaci.VolumeMount{ + Name: &container.VolumeMounts[i].Name, + MountPath: &container.VolumeMounts[i].MountPath, + ReadOnly: &container.VolumeMounts[i].ReadOnly, + }) + } + return &volumeMounts +} + +//get EnvironmentVariables declared on Container as []aci.EnvironmentVariable +func (p *ACIProvider) getEnvironmentVariables(container *v1.Container) *[]azaci.EnvironmentVariable { + environmentVariable := make([]azaci.EnvironmentVariable, 0, len(container.Env)) + for i := range container.Env { + if container.Env[i].Value != "" { + envVar := getACIEnvVar(container.Env[i]) + environmentVariable = append(environmentVariable, envVar) + } + } + return &environmentVariable +} + +//get InitContainers defined in Pod as []aci.InitContainerDefinition +func (p *ACIProvider) getInitContainers(ctx context.Context, pod *v1.Pod) ([]azaci.InitContainerDefinition, error) { + initContainers := make([]azaci.InitContainerDefinition, 0, len(pod.Spec.InitContainers)) + for i, initContainer := range pod.Spec.InitContainers { + err := p.verifyContainer(&initContainer) + if err != nil { + log.G(ctx).Errorf("couldn't verify container %v", err) + return nil, err + } + + if initContainer.Ports != nil { + log.G(ctx).Errorf("azure container instances initcontainers do not support ports") + return nil, errdefs.InvalidInput("azure container instances initContainers do not support ports") + } + if initContainer.Resources.Requests != nil { + log.G(ctx).Errorf("azure container instances initcontainers do not support resources requests") + return nil, errdefs.InvalidInput("azure container instances initContainers do not support resources requests") + } + if initContainer.Resources.Limits != nil { + log.G(ctx).Errorf("azure container instances initcontainers do not support resources limits") + return nil, errdefs.InvalidInput("azure container instances initContainers do not support resources limits") + } + if initContainer.LivenessProbe != nil { + log.G(ctx).Errorf("azure container instances initcontainers do not support livenessProbe") + return nil, errdefs.InvalidInput("azure container instances initContainers do not support livenessProbe") + } + if initContainer.ReadinessProbe != nil { + log.G(ctx).Errorf("azure container instances initcontainers do not support readinessProbe") + return nil, errdefs.InvalidInput("azure container instances initContainers do not support readinessProbe") + } + + newInitContainer := azaci.InitContainerDefinition{ + Name: &pod.Spec.InitContainers[i].Name, + InitContainerPropertiesDefinition: &azaci.InitContainerPropertiesDefinition { + Image: &pod.Spec.InitContainers[i].Image, + Command: p.getCommand(&pod.Spec.InitContainers[i]), + VolumeMounts: p.getVolumeMounts(&pod.Spec.InitContainers[i]), + EnvironmentVariables: p.getEnvironmentVariables(&pod.Spec.InitContainers[i]), + }, + } + + initContainers = append(initContainers, newInitContainer) + } + return initContainers, nil +} + func (p *ACIProvider) getContainers(pod *v1.Pod) (*[]azaci.Container, error) { containers := make([]azaci.Container, 0, len(pod.Spec.Containers)) diff --git a/pkg/provider/aci_init_container_test.go b/pkg/provider/aci_init_container_test.go new file mode 100644 index 00000000..e9de3d13 --- /dev/null +++ b/pkg/provider/aci_init_container_test.go @@ -0,0 +1,206 @@ +package provider + + +import ( + "context" + "testing" + + "github.com/virtual-kubelet/virtual-kubelet/errdefs" + "github.com/golang/mock/gomock" + "github.com/virtual-kubelet/azure-aci/pkg/client" + "github.com/virtual-kubelet/node-cli/manager" + "gotest.tools/assert" + is "gotest.tools/assert/cmp" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/api/resource" +) + +func TestCreatePodWithInitContainers(t *testing.T) { + + initContainerName1 := "init-container-1" + initContainerName2 := "init-container-2" + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + aciMocks := createNewACIMock() + aciMocks.MockCreateContainerGroup = func(ctx context.Context, resourceGroup, podNS, podName string, cg *client.ContainerGroupWrapper) error { + containers := *cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Containers + initContainers := *cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.InitContainers + assert.Check(t, cg != nil, "Container group is nil") + assert.Check(t, containers != nil, "Containers should not be nil") + assert.Check(t, initContainers != nil, "Container group is nil") + assert.Check(t, is.Equal(len(containers), 2), "1 Container is expected") + assert.Check(t, is.Equal(len(initContainers), 2), "2 init containers are expected") + assert.Check(t, initContainers[0].VolumeMounts != nil, "Volume mount should be present") + assert.Check(t, initContainers[0].EnvironmentVariables != nil, "Volume mount should be present") + assert.Check(t, initContainers[0].Command != nil, "Command mount should be present") + assert.Check(t, initContainers[0].Image != nil, "Image should be present") + assert.Check(t, *initContainers[0].Name == initContainerName1, "Name should be correct") + assert.Check(t, *initContainers[1].Name == initContainerName2, "Name should be correct") + + return nil + } + + + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Namespace: podNamespace, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "container-name-01", + Image: "alpine", + }, + { + Name: "container-name-02", + Image: "alpine", + }, + }, + }, + } + cases := []struct { + description string + initContainers []v1.Container + expectedError error + }{ + { + description: "Init Containers with Supported fields", + expectedError: nil, + initContainers: []v1.Container{ + v1.Container{ + Name: initContainerName1, + Image: "alpine", + VolumeMounts: []v1.VolumeMount{ + v1.VolumeMount{ + Name: "fakeVolumeName", + MountPath: "/mnt/azure", + }, + }, + Command: []string{"/bin/bash"}, + Args: []string{"-c echo test"}, + Env: []v1.EnvVar{ + v1.EnvVar{ + Name: "TEST_ENV", + Value: "testvalue", + }, + }, + }, + v1.Container{ + Name: initContainerName2, + Image: "alpine", + }, + }, + }, + { + description: "Init Containers with ports", + initContainers: []v1.Container{ + v1.Container{ + Name: "initContainer 01", + Image: "alpine", + Ports: []v1.ContainerPort{ + v1.ContainerPort{ + Name: "http", + ContainerPort: 80, + Protocol: "TCP", + }, + }, + }, + }, + expectedError: errdefs.InvalidInput("azure container instances initContainers do not support ports"), + }, + { + description: "Init Containers with liveness probe", + initContainers: []v1.Container{ + v1.Container{ + Name: "initContainer 01", + LivenessProbe: &v1.Probe{ + Handler: v1.Handler{ + HTTPGet: &v1.HTTPGetAction{ + Port: intstr.FromString("http"), + Path: "/", + }, + }, + InitialDelaySeconds: 10, + PeriodSeconds: 5, + TimeoutSeconds: 60, + SuccessThreshold: 3, + FailureThreshold: 5, + }, + }, + }, + expectedError: errdefs.InvalidInput("azure container instances initContainers do not support livenessProbe"), + }, + { + description: "Init Containers with readiness probe", + initContainers: []v1.Container{ + v1.Container{ + Name: "initContainer 01", + ReadinessProbe: &v1.Probe{ + Handler: v1.Handler{ + HTTPGet: &v1.HTTPGetAction{ + Port: intstr.FromInt(8080), + Path: "/", + }, + }, + InitialDelaySeconds: 10, + PeriodSeconds: 5, + TimeoutSeconds: 60, + SuccessThreshold: 3, + FailureThreshold: 5, + }, + }, + }, + expectedError: errdefs.InvalidInput("azure container instances initContainers do not support readinessProbe"), + }, + { + description: "Init Containers with resource request", + initContainers: []v1.Container{ + { + Name: "initContainer 01", + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + "cpu": resource.MustParse("1.981"), + "memory": resource.MustParse("3.49G"), + }, + Limits: v1.ResourceList{ + gpuResourceName: resource.MustParse("10"), + }, + }, + }, + }, + expectedError: errdefs.InvalidInput("azure container instances initContainers do not support resources requests"), + }, + } + for _, tc := range cases { + t.Run(tc.description, func(t *testing.T) { + + resourceManager, err := manager.NewResourceManager( + NewMockPodLister(mockCtrl), + NewMockSecretLister(mockCtrl), + NewMockConfigMapLister(mockCtrl), + NewMockServiceLister(mockCtrl), + NewMockPersistentVolumeClaimLister(mockCtrl), + NewMockPersistentVolumeLister(mockCtrl)) + if err != nil { + t.Fatal("Unable to prepare the mocks for resourceManager", err) + } + + provider, err := createTestProvider(aciMocks, resourceManager) + if err != nil { + t.Fatal("Unable to create test provider", err) + } + + pod.Spec.InitContainers = tc.initContainers + err = provider.CreatePod(context.Background(), pod) + + // check that the correct error is returned + if tc.expectedError != nil && err != tc.expectedError { + assert.Equal(t, tc.expectedError.Error(), err.Error(), "expected error and actual error don't match") + } + }) + } +} diff --git a/pkg/provider/aci_volumes_test.go b/pkg/provider/aci_volumes_test.go index f5a02b1f..ef93e38e 100644 --- a/pkg/provider/aci_volumes_test.go +++ b/pkg/provider/aci_volumes_test.go @@ -34,6 +34,7 @@ func TestCreatedPodWithAzureFilesVolume(t *testing.T) { azureFileVolumeName1 := "azurefile1" azureFileVolumeName2 := "azurefile2" fakeSecretName := "fake-secret" + initContainerName := "init-container" mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() @@ -41,6 +42,7 @@ func TestCreatedPodWithAzureFilesVolume(t *testing.T) { aciMocks := createNewACIMock() aciMocks.MockCreateContainerGroup = func(ctx context.Context, resourceGroup, podNS, podName string, cg *client.ContainerGroupWrapper) error { containers := *cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Containers + initContainers := *cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.InitContainers assert.Check(t, cg != nil, "Container group is nil") assert.Check(t, containers != nil, "Containers should not be nil") assert.Check(t, is.Equal(1, len(containers)), "1 Container is expected") @@ -50,6 +52,11 @@ func TestCreatedPodWithAzureFilesVolume(t *testing.T) { assert.Check(t, is.Equal(fakeShareName1, *(*cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Volumes)[1].AzureFile.ShareName), "volume share name is not matched") assert.Check(t, is.Equal(azureFileVolumeName2, *(*cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Volumes)[2].Name), "volume name is not matched") assert.Check(t, is.Equal(fakeShareName2, *(*cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Volumes)[2].AzureFile.ShareName), "volume share name is not matched") + assert.Check(t, initContainers[0].VolumeMounts != nil, "Volume mount should be present") + assert.Check(t, initContainers[0].EnvironmentVariables != nil, "Volume mount should be present") + assert.Check(t, initContainers[0].Command != nil, "Command mount should be present") + assert.Check(t, initContainers[0].Image != nil, "Image should be present") + assert.Check(t, *initContainers[0].Name == initContainerName, "Name should be correct") return nil } @@ -144,8 +151,27 @@ func TestCreatedPodWithAzureFilesVolume(t *testing.T) { mockSecretLister := NewMockSecretLister(mockCtrl) pod := testsutil.CreatePodObj(podName, podNamespace) - pod.Spec.Containers[0].VolumeMounts = fakeVolumeMount + pod.Spec.InitContainers = []v1.Container{ + v1.Container{ + Name: initContainerName, + Image: "alpine", + VolumeMounts: []v1.VolumeMount{ + v1.VolumeMount{ + Name: "fakeVolume", + MountPath: "/mnt/azure", + }, + }, + Command: []string{"/bin/bash"}, + Args: []string{"-c echo test"}, + Env: []v1.EnvVar{ + v1.EnvVar{ + Name: "TEST_ENV", + Value: "testvalue", + }, + }, + }, + } tc.callSecretMocks(mockSecretLister) pod.Spec.Volumes = tc.volumes