From 4581d7ba41e3ff519458063f9d3d1ea80f5d69ce Mon Sep 17 00:00:00 2001 From: Seth Heidkamp <61526534+sheidkamp@users.noreply.github.com> Date: Thu, 30 Jan 2025 11:55:28 -0500 Subject: [PATCH] Add support for TLS for xDS with Gloo Gateway (#10582) Co-authored-by: David Jumani Co-authored-by: Jenny Shu <28537278+jenshu@users.noreply.github.com> --- .github/workflows/pr-kubernetes-tests.yaml | 4 +- .../v1.19.0-beta5/k8s-gw-mtls-deployer.yaml | 6 + install/helm/gloo/generate/values.go | 1 - .../gloo/templates/1-gloo-deployment.yaml | 4 + install/helm/gloo/templates/44-rbac.yaml | 7 +- projects/gateway2/controller/controller.go | 30 ++- projects/gateway2/controller/start.go | 11 +- projects/gateway2/deployer/deployer.go | 78 +++++++- projects/gateway2/deployer/deployer_test.go | 187 +++++++++++++----- projects/gateway2/deployer/values.go | 15 ++ .../templates/gateway/proxy-deployment.yaml | 76 ++++++- projects/gateway2/setup/ggv2setup.go | 17 +- projects/gateway2/wellknown/controller.go | 3 + .../features/gloomtls/gloomtls_edge_suite.go | 39 +--- .../gloomtls/gloomtls_k8s_gateway_suite.go | 114 +++++++++++ .../features/gloomtls/testdata/certgen.yaml | 42 ++++ .../gloomtls/testdata/hello-route.yaml | 13 ++ .../gloomtls/testdata/route-with-service.yaml | 27 +++ .../gloomtls/testdata/service-for-route.yaml | 26 +++ .../kubernetes/e2e/features/gloomtls/types.go | 31 +++ .../e2e/tests/gloomtls_k8s_gw_test.go | 54 +++++ .../e2e/tests/gloomtls_k8s_gw_tests.go | 12 ++ .../gloomtls-edge-gateway-test-helm.yaml | 3 + tilt-settings.yaml | 5 +- 24 files changed, 711 insertions(+), 94 deletions(-) create mode 100644 changelog/v1.19.0-beta5/k8s-gw-mtls-deployer.yaml create mode 100644 test/kubernetes/e2e/features/gloomtls/gloomtls_k8s_gateway_suite.go create mode 100644 test/kubernetes/e2e/features/gloomtls/testdata/certgen.yaml create mode 100644 test/kubernetes/e2e/features/gloomtls/testdata/hello-route.yaml create mode 100644 test/kubernetes/e2e/features/gloomtls/testdata/route-with-service.yaml create mode 100644 test/kubernetes/e2e/features/gloomtls/testdata/service-for-route.yaml create mode 100644 test/kubernetes/e2e/tests/gloomtls_k8s_gw_test.go create mode 100644 test/kubernetes/e2e/tests/gloomtls_k8s_gw_tests.go diff --git a/.github/workflows/pr-kubernetes-tests.yaml b/.github/workflows/pr-kubernetes-tests.yaml index 79ddaf794fe..eea0e4517be 100644 --- a/.github/workflows/pr-kubernetes-tests.yaml +++ b/.github/workflows/pr-kubernetes-tests.yaml @@ -85,10 +85,10 @@ jobs: go-test-args: '-v -timeout=30m' go-test-run-regex: '^TestDiscoveryWatchlabels$$|^TestK8sGatewayNoValidation$$|^TestHelm$$|^TestHelmSettings$$|^TestK8sGatewayAws$$|^TestK8sGateway$$/^HTTPRouteServices$$|^TestK8sGateway$$/^TCPRouteServices$$|^TestZeroDowntimeRollout$$' - # Dec 4, 2024: 13 minutes + # Dec 4, 2024: 16 minutes - cluster-name: 'cluster-seven' go-test-args: '-v -timeout=25m' - go-test-run-regex: '^TestK8sGateway$$/^CRDCategories$$|^TestK8sGateway$$/^Metrics$$|^TestGloomtlsGatewayEdgeGateway$$|^TestWatchNamespaceSelector$$' + go-test-run-regex: '^TestK8sGateway$$/^CRDCategories$$|^TestK8sGateway$$/^Metrics$$|^TestGloomtlsGatewayEdgeGateway$$|^TestGloomtlsGatewayK8sGateway$$|^TestWatchNamespaceSelector$$' # In our PR tests, we run the suite of tests using the upper ends of versions that we claim to support # The versions should mirror: https://docs.solo.io/gloo-edge/latest/reference/support/ diff --git a/changelog/v1.19.0-beta5/k8s-gw-mtls-deployer.yaml b/changelog/v1.19.0-beta5/k8s-gw-mtls-deployer.yaml new file mode 100644 index 00000000000..4a7e3a86fac --- /dev/null +++ b/changelog/v1.19.0-beta5/k8s-gw-mtls-deployer.yaml @@ -0,0 +1,6 @@ +changelog: + - type: NEW_FEATURE + issueLink: https://github.com/solo-io/solo-projects/issues/6210 + resolvesIssue: false + description: >- + Add support for xDS over mTLS for communication between the Gloo pod and the Kubernetes Gateway proxies. This can be enabled by setting the 'global.glooMtls.enabled' helm value to true. diff --git a/install/helm/gloo/generate/values.go b/install/helm/gloo/generate/values.go index ad92ad72008..4e317a78ae6 100644 --- a/install/helm/gloo/generate/values.go +++ b/install/helm/gloo/generate/values.go @@ -340,7 +340,6 @@ type GatewayParameters struct { AIExtension *GatewayParamsAIExtension `json:"aiExtension,omitempty" desc:"Config used to manage the Gloo Gateway AI extension."` FloatingUserId *bool `json:"floatingUserId,omitempty" desc:"If true, allows the cluster to dynamically assign a user ID for the processes running in the container. Default is false."` PodTemplate *GatewayParamsPodTemplate `json:"podTemplate,omitempty" desc:"The template used to generate the gatewayParams pod"` - // TODO(npolshak): Add support for GlooMtls } // GatewayProxyPodTemplate contains the Helm API available to configure the PodTemplate on the gateway Deployment diff --git a/install/helm/gloo/templates/1-gloo-deployment.yaml b/install/helm/gloo/templates/1-gloo-deployment.yaml index db7d3f7874f..2b27ba17c87 100644 --- a/install/helm/gloo/templates/1-gloo-deployment.yaml +++ b/install/helm/gloo/templates/1-gloo-deployment.yaml @@ -273,6 +273,10 @@ spec: - name: HEADER_SECRET_REF_NS_MATCHES_US value: "true" {{- end}} + {{- if .Values.global.glooMtls.enabled }} + - name: GLOO_MTLS_SDS_ENABLED + value: "true" + {{- end }} {{- if not .Values.global.glooMtls.enabled }} readinessProbe: tcpSocket: diff --git a/install/helm/gloo/templates/44-rbac.yaml b/install/helm/gloo/templates/44-rbac.yaml index 6934b3925e4..68a4530a9fb 100644 --- a/install/helm/gloo/templates/44-rbac.yaml +++ b/install/helm/gloo/templates/44-rbac.yaml @@ -22,9 +22,14 @@ rules: - services - pods - nodes - - secrets - namespaces verbs: ["get", "list", "watch"] +- apiGroups: + - "" + resources: + - secrets + {{/* This is needed as the gateway deployer would need to create / patch the mtls certs if enabled */}} + verbs: ["get", "list", "watch", "create", "patch"] - apiGroups: - "discovery.k8s.io" resources: diff --git a/projects/gateway2/controller/controller.go b/projects/gateway2/controller/controller.go index 3f72dc7c2d3..941a5f6610f 100644 --- a/projects/gateway2/controller/controller.go +++ b/projects/gateway2/controller/controller.go @@ -188,6 +188,11 @@ func (c *controllerBuilder) addHttpLisOptIndexes(ctx context.Context) error { }) } +func (c *controllerBuilder) shouldWatchSecrets() bool { + // watch for secrets if mtls is enabled + return c.cfg.ControlPlane.GlooMtlsEnabled +} + func (c *controllerBuilder) watchGw(ctx context.Context) error { // setup a deployer log := log.FromContext(ctx) @@ -224,8 +229,31 @@ func (c *controllerBuilder) watchGw(ctx context.Context) error { ), )) - // watch for changes in GatewayParameters cli := c.cfg.Mgr.GetClient() + + // watch for secrets if needed + if c.shouldWatchSecrets() { + buildr.Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc( + func(ctx context.Context, obj client.Object) []reconcile.Request { + var reqs []reconcile.Request + if obj.GetName() == wellknown.GlooMtlsCertName && obj.GetNamespace() == c.cfg.ControlPlane.Namespace { + var gwList apiv1.GatewayList + err := cli.List(ctx, &gwList, client.InNamespace(corev1.NamespaceAll)) + if err != nil { + log.Error(err, "could not list Gateways", "namespace", corev1.NamespaceAll) + return reqs + } + + for _, gw := range gwList.Items { + reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKey{Namespace: gw.Namespace, Name: gw.Name}}) + } + return reqs + } + return reqs + })) + } + + // watch for changes in GatewayParameters buildr.Watches(&v1alpha1.GatewayParameters{}, handler.EnqueueRequestsFromMapFunc( func(ctx context.Context, obj client.Object) []reconcile.Request { gwpName := obj.GetName() diff --git a/projects/gateway2/controller/start.go b/projects/gateway2/controller/start.go index 068838e6e9b..ead63d118ed 100644 --- a/projects/gateway2/controller/start.go +++ b/projects/gateway2/controller/start.go @@ -8,6 +8,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/config" glooschemes "github.com/solo-io/gloo/pkg/schemes" + "github.com/solo-io/gloo/pkg/utils/namespaces" "github.com/solo-io/go-utils/contextutils" "k8s.io/apimachinery/pkg/util/sets" @@ -77,6 +78,8 @@ type StartConfig struct { InitialSettings *glookubev1.Settings Settings krt.Singleton[glookubev1.Settings] + GlooMtlsEnabled bool + Debugger *krt.DebugHandler } @@ -221,7 +224,7 @@ func (c *ControllerBuilder) Start(ctx context.Context) error { return ctx.Err() } - logger.Infow("got xds address for deployer", uzap.String("xds_host", xdsHost), uzap.Int32("xds_port", xdsPort)) + logger.Infow("got xds address for deployer", uzap.String("xds_host", xdsHost), uzap.Int32("xds_port", xdsPort), uzap.Any("glooMtlsEnabled", c.cfg.GlooMtlsEnabled)) integrationEnabled := c.cfg.InitialSettings.Spec.GetGloo().GetIstioOptions().GetEnableIntegration().GetValue() @@ -255,8 +258,10 @@ func (c *ControllerBuilder) Start(ctx context.Context) error { ControllerName: wellknown.GatewayControllerName, AutoProvision: AutoProvision, ControlPlane: deployer.ControlPlaneInfo{ - XdsHost: xdsHost, - XdsPort: xdsPort, + XdsHost: xdsHost, + XdsPort: xdsPort, + GlooMtlsEnabled: c.cfg.GlooMtlsEnabled, + Namespace: namespaces.GetPodNamespace(), }, // TODO pass in the settings so that the deloyer can register to it for changes. IstioIntegrationEnabled: integrationEnabled, diff --git a/projects/gateway2/deployer/deployer.go b/projects/gateway2/deployer/deployer.go index 1ee1632a781..e6fcd0fa80c 100644 --- a/projects/gateway2/deployer/deployer.go +++ b/projects/gateway2/deployer/deployer.go @@ -25,6 +25,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/yaml" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" @@ -53,6 +54,11 @@ type Deployer struct { type ControlPlaneInfo struct { XdsHost string XdsPort int32 + // The data in this struct is static, so is a good place to keep track of if mtls is enabled + // and a bad place to store the actual mtls secret data + GlooMtlsEnabled bool + // We could lookup the pod namespace from the env, but it's cleaner to pass it in + Namespace string } type AwsInfo struct { @@ -103,17 +109,21 @@ func (d *Deployer) GetGvksToWatch(ctx context.Context) ([]schema.GroupVersionKin // as we only care about the GVKs of the rendered resources) // - the minimal values that render all the proxy resources (HPA is not included because it's not // fully integrated/working at the moment) + // - a flag to indicate whether mtls is enabled, so we can render the secret if needed // // Note: another option is to hardcode the GVKs here, but rendering the helm chart is a // _slightly_ more dynamic way of getting the GVKs. It isn't a perfect solution since if // we add more resources to the helm chart that are gated by a flag, we may forget to // update the values here to enable them. + // Currently the only resource that is gated by a flag is the mtls secret. + emptyGw := &api.Gateway{ ObjectMeta: metav1.ObjectMeta{ Name: "default", Namespace: "default", }, } + // TODO(Law): these must be set explicitly as we don't have defaults for them // and the internal template isn't robust enough. // This should be empty eventually -- the template must be resilient against nil-pointers @@ -124,6 +134,12 @@ func (d *Deployer) GetGvksToWatch(ctx context.Context) ([]schema.GroupVersionKin "enabled": false, }, "image": map[string]any{}, + // Render the secret based on the mtls flag so we can watch it. + // This is an exception to the "TODO" above as this is not protection against nil-pointers, + // it is determining which resources to render based on ControlPlane configuration. + "glooMtls": map[string]any{ + "renderSecret": d.inputs.ControlPlane.GlooMtlsEnabled, + }, }, } @@ -131,6 +147,7 @@ func (d *Deployer) GetGvksToWatch(ctx context.Context) ([]schema.GroupVersionKin if err != nil { return nil, err } + var ret []schema.GroupVersionKind for _, obj := range objs { gvk := obj.GetObjectKind().GroupVersionKind() @@ -255,7 +272,7 @@ func (d *Deployer) getGatewayClassFromGateway(ctx context.Context, gw *api.Gatew return gwc, nil } -func (d *Deployer) getValues(gw *api.Gateway, gwParam *v1alpha1.GatewayParameters) (*helmConfig, error) { +func (d *Deployer) getValues(ctx context.Context, gw *api.Gateway, gwParam *v1alpha1.GatewayParameters) (*helmConfig, error) { // construct the default values vals := &helmConfig{ Gateway: &helmGateway{ @@ -361,9 +378,61 @@ func (d *Deployer) getValues(gw *api.Gateway, gwParam *v1alpha1.GatewayParameter gateway.Stats = getStatsValues(statsConfig) + // mtls values + gateway.GlooMtls, err = d.getHelmMtlsConfig(ctx) + if err != nil { + return nil, err + } + return vals, nil } +func (d *Deployer) getHelmMtlsConfig(ctx context.Context) (*helmMtlsConfig, error) { + + if !d.inputs.ControlPlane.GlooMtlsEnabled { + return &helmMtlsConfig{ + Enabled: ptr.To(false), + }, nil + } + + helmTls, err := d.getHelmTlsSecretData(ctx) + + if err != nil { + return nil, err + } + + return &helmMtlsConfig{ + Enabled: ptr.To(true), + TlsSecret: helmTls, + }, nil +} + +// getHelmTlsSecretData builds a helmTls object built from the gloo-mtls-certs secret data, which it fetches +// This function does not check if mtls is enabled, and a missing secret will return an error via getGlooMtlsCertsSecret +func (d *Deployer) getHelmTlsSecretData(ctx context.Context) (*helmTlsSecretData, error) { + + mtlsSecret := &corev1.Secret{} + mtlsSecretNns := types.NamespacedName{ + Name: wellknown.GlooMtlsCertName, + Namespace: d.inputs.ControlPlane.Namespace, + } + err := d.cli.Get(ctx, mtlsSecretNns, mtlsSecret) + + if err != nil { + return nil, eris.Wrap(err, "failed to get gloo mtls secret") + } + + if mtlsSecret.Type != corev1.SecretTypeTLS { + return nil, eris.New(fmt.Sprintf("unexpected secret type, expected %s and got %s", corev1.SecretTypeTLS, mtlsSecret.Type)) + } + + return &helmTlsSecretData{ + TlsCert: mtlsSecret.Data[corev1.TLSCertKey], + TlsKey: mtlsSecret.Data[corev1.TLSPrivateKeyKey], + CaCert: mtlsSecret.Data[corev1.ServiceAccountRootCAKey], + }, nil +} + // Render relies on a `helm install` to render the Chart with the injected values // It returns the list of Objects that are rendered, and an optional error if rendering failed, // or converting the rendered manifests to objects failed. @@ -392,6 +461,7 @@ func (d *Deployer) Render(name, ns string, vals map[string]any) ([]client.Object if err != nil { return nil, fmt.Errorf("failed to convert helm manifest yaml to objects for gateway %s.%s: %w", ns, name, err) } + return objs, nil } @@ -405,6 +475,8 @@ func (d *Deployer) Render(name, ns string, vals map[string]any) ([]client.Object // // * returns the objects to be deployed by the caller func (d *Deployer) GetObjsToDeploy(ctx context.Context, gw *api.Gateway) ([]client.Object, error) { + logger := log.FromContext(ctx) + gwParam, err := d.getGatewayParametersForGateway(ctx, gw) if err != nil { return nil, err @@ -414,9 +486,7 @@ func (d *Deployer) GetObjsToDeploy(ctx context.Context, gw *api.Gateway) ([]clie return nil, nil } - logger := log.FromContext(ctx) - - vals, err := d.getValues(gw, gwParam) + vals, err := d.getValues(ctx, gw, gwParam) if err != nil { return nil, fmt.Errorf("failed to get values to render objects for gateway %s.%s: %w", gw.GetNamespace(), gw.GetName(), err) } diff --git a/projects/gateway2/deployer/deployer_test.go b/projects/gateway2/deployer/deployer_test.go index dc8512b3671..7063928e412 100644 --- a/projects/gateway2/deployer/deployer_test.go +++ b/projects/gateway2/deployer/deployer_test.go @@ -25,6 +25,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/yaml" "k8s.io/utils/pointer" @@ -106,6 +107,17 @@ func (objs *clientObjects) findConfigMap(namespace, name string) *corev1.ConfigM return nil } +func (objs *clientObjects) findSecret(namespace, name string) *corev1.Secret { + for _, obj := range *objs { + if secret, ok := obj.(*corev1.Secret); ok { + if secret.Name == name && secret.Namespace == namespace { + return secret + } + } + } + return nil +} + func (objs *clientObjects) getEnvoyConfig(namespace, name string) *testBootstrap { cm := objs.findConfigMap(namespace, name).Data var bootstrapCfg testBootstrap @@ -118,13 +130,19 @@ func proxyName(name string) string { return fmt.Sprintf("gloo-proxy-%s", name) } +func mtlsSecretData() map[string][]byte { + return map[string][]byte{ + "tls.crt": []byte("cert"), + "tls.key": []byte("key"), + "ca.crt": []byte("ca"), + } +} + var _ = Describe("Deployer", func() { const ( defaultNamespace = "default" ) var ( - d *deployer.Deployer - defaultGatewayClass = func() *api.GatewayClass { return &api.GatewayClass{ ObjectMeta: metav1.ObjectMeta{ @@ -237,6 +255,12 @@ var _ = Describe("Deployer", func() { } } + defaultGatewayParamsWithSds = func() *gw2_v1alpha1.GatewayParameters { + gwp := defaultGatewayParams() + gwp.Spec.Kube.SdsContainer = defaultSdsContainer() + return gwp + } + defaultDeploymentName = proxyName(defaultGateway().Name) defaultConfigMapName = defaultDeploymentName defaultServiceName = defaultDeploymentName @@ -413,32 +437,60 @@ var _ = Describe("Deployer", func() { }) }) - Context("special cases", func() { + Context("GetGvksToWatch", func() { var gwc *api.GatewayClass + BeforeEach(func() { gwc = defaultGatewayClass() - var err error - - d, err = deployer.NewDeployer(newFakeClientWithObjs(gwc, defaultGatewayParams()), &deployer.Inputs{ - ControllerName: wellknown.GatewayControllerName, - Dev: false, - ControlPlane: deployer.ControlPlaneInfo{ - XdsHost: "something.cluster.local", XdsPort: 1234, - }, - }) - Expect(err).NotTo(HaveOccurred()) }) - It("should get gvks", func() { + DescribeTable("gets correct gvks", func(inputs *deployer.Inputs, expectedGvks []schema.GroupVersionKind) { + d, err := deployer.NewDeployer(newFakeClientWithObjs(gwc, defaultGatewayParams()), inputs) + Expect(err).NotTo(HaveOccurred()) gvks, err := d.GetGvksToWatch(context.Background()) Expect(err).NotTo(HaveOccurred()) - Expect(gvks).To(HaveLen(4)) - Expect(gvks).To(ConsistOf( - wellknownkube.DeploymentGVK, - wellknownkube.ServiceGVK, - wellknownkube.ServiceAccountGVK, - wellknownkube.ConfigMapGVK, + Expect(gvks).To(HaveLen(len(expectedGvks))) + Expect(gvks).To(ConsistOf(expectedGvks)) + + }, + Entry("glooMtls enabled", + &deployer.Inputs{ + ControllerName: wellknown.GatewayControllerName, + Dev: false, + ControlPlane: deployer.ControlPlaneInfo{ + XdsHost: "something.cluster.local", XdsPort: 1234, + GlooMtlsEnabled: true, + }, + }, + []schema.GroupVersionKind{ + wellknownkube.DeploymentGVK, + wellknownkube.ServiceGVK, + wellknownkube.ServiceAccountGVK, + wellknownkube.ConfigMapGVK, + wellknownkube.SecretGVK, + }), + Entry("glooMtls disabled", + &deployer.Inputs{ + ControllerName: wellknown.GatewayControllerName, + Dev: false, + ControlPlane: deployer.ControlPlaneInfo{ + XdsHost: "something.cluster.local", XdsPort: 1234, + }, + }, + []schema.GroupVersionKind{ + wellknownkube.DeploymentGVK, + wellknownkube.ServiceGVK, + wellknownkube.ServiceAccountGVK, + wellknownkube.ConfigMapGVK, + }, )) + + }) + + Context("special cases", func() { + var gwc *api.GatewayClass + BeforeEach(func() { + gwc = defaultGatewayClass() }) It("support segmenting by release", func() { @@ -549,6 +601,13 @@ var _ = Describe("Deployer", func() { return inp } + mtlsEnabledInputs = func() *deployer.Inputs { + inp := defaultDeployerInputs() + inp.ControlPlane.GlooMtlsEnabled = true + inp.ControlPlane.Namespace = defaultNamespace + return inp + } + defaultGatewayParamsOverride = func() *gw2_v1alpha1.GatewayParameters { return &gw2_v1alpha1.GatewayParameters{ TypeMeta: metav1.TypeMeta{ @@ -801,11 +860,20 @@ var _ = Describe("Deployer", func() { } } - validateGatewayParametersPropagation = func(objs clientObjects, gwp *gw2_v1alpha1.GatewayParameters) error { + mtlsEnabled = true + mtlsDisabled = false + + validateGatewayParametersPropagation = func(objs clientObjects, gwp *gw2_v1alpha1.GatewayParameters, mtlsEnabled bool) error { expectedGwp := gwp.Spec.Kube Expect(objs).NotTo(BeEmpty()) - // Check we have Deployment, ConfigMap, ServiceAccount, Service - Expect(objs).To(HaveLen(4)) + if mtlsEnabled { + // Check we have Deployment, ConfigMap, ServiceAccount, Service, Secret + Expect(objs).To(HaveLen(5)) + } else { + // Check we have Deployment, ConfigMap, ServiceAccount, Service + Expect(objs).To(HaveLen(4)) + } + dep := objs.findDeployment(defaultNamespace, defaultDeploymentName) Expect(dep).ToNot(BeNil()) Expect(dep.Spec.Replicas).ToNot(BeNil()) @@ -847,6 +915,13 @@ var _ = Describe("Deployer", func() { cm := objs.findConfigMap(defaultNamespace, defaultConfigMapName) Expect(cm).ToNot(BeNil()) + if mtlsEnabled { + secret := objs.findSecret(defaultNamespace, wellknown.GlooMtlsCertName) + Expect(secret).ToNot(BeNil()) + Expect(secret.Type).To(Equal(corev1.SecretTypeTLS)) + Expect(secret.Data).To(BeEquivalentTo(mtlsSecretData())) + } + logLevelsMap := expectedGwp.EnvoyContainer.Bootstrap.ComponentLogLevels levels := []types.GomegaMatcher{} for k, v := range logLevelsMap { @@ -1154,7 +1229,16 @@ var _ = Describe("Deployer", func() { overrideGwp = &gw2_v1alpha1.GatewayParameters{} } - d, err := deployer.NewDeployer(newFakeClientWithObjs(gwc, defaultGwp, overrideGwp), inp.dInputs) + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: wellknown.GlooMtlsCertName, + Namespace: defaultNamespace, + }, + Type: corev1.SecretTypeTLS, + Data: mtlsSecretData(), + } + + d, err := deployer.NewDeployer(newFakeClientWithObjs(gwc, defaultGwp, overrideGwp, secret), inp.dInputs) if checkErr(err, expected.newDeployerErr) { return } @@ -1173,7 +1257,7 @@ var _ = Describe("Deployer", func() { defaultGwp: defaultGatewayParams(), }, &expectedOutput{ validationFunc: func(objs clientObjects, inp *input) error { - return validateGatewayParametersPropagation(objs, defaultGatewayParams()) + return validateGatewayParametersPropagation(objs, defaultGatewayParams(), mtlsDisabled) }, }), Entry("GatewayParameters overrides", &input{ @@ -1183,7 +1267,7 @@ var _ = Describe("Deployer", func() { overrideGwp: defaultGatewayParamsOverride(), }, &expectedOutput{ validationFunc: func(objs clientObjects, inp *input) error { - return validateGatewayParametersPropagation(objs, mergedGatewayParams()) + return validateGatewayParametersPropagation(objs, mergedGatewayParams(), mtlsDisabled) }, }), Entry("Fully defined GatewayParameters", &input{ @@ -1473,6 +1557,15 @@ var _ = Describe("Deployer", func() { return nil }, }), + Entry("glooMtls enabled", &input{ + dInputs: mtlsEnabledInputs(), + gw: defaultGateway(), + defaultGwp: defaultGatewayParamsWithSds(), + }, &expectedOutput{ + validationFunc: func(objs clientObjects, inp *input) error { + return validateGatewayParametersPropagation(objs, defaultGatewayParamsWithSds(), mtlsEnabled) + }, + }), ) }) }) @@ -1485,6 +1578,28 @@ func newFakeClientWithObjs(objs ...client.Object) client.Client { Build() } +func defaultSdsContainer() *gw2_v1alpha1.SdsContainer { + return &gw2_v1alpha1.SdsContainer{ + Image: &gw2_v1alpha1.Image{ + Registry: ptr.To("sds-registry"), + Repository: ptr.To("sds-repository"), + Tag: ptr.To("sds-tag"), + Digest: ptr.To("sds-digest"), + PullPolicy: ptr.To(corev1.PullAlways), + }, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(int64(222)), + }, + Resources: &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{"cpu": resource.MustParse("201m")}, + Requests: corev1.ResourceList{"cpu": resource.MustParse("203m")}, + }, + Bootstrap: &gw2_v1alpha1.SdsBootstrap{ + LogLevel: ptr.To("debug"), + }, + } +} + func fullyDefinedGatewayParameters(name, namespace string) *gw2_v1alpha1.GatewayParameters { return &gw2_v1alpha1.GatewayParameters{ TypeMeta: metav1.TypeMeta{ @@ -1524,25 +1639,7 @@ func fullyDefinedGatewayParameters(name, namespace string) *gw2_v1alpha1.Gateway Requests: corev1.ResourceList{"cpu": resource.MustParse("103m")}, }, }, - SdsContainer: &gw2_v1alpha1.SdsContainer{ - Image: &gw2_v1alpha1.Image{ - Registry: ptr.To("sds-registry"), - Repository: ptr.To("sds-repository"), - Tag: ptr.To("sds-tag"), - Digest: ptr.To("sds-digest"), - PullPolicy: ptr.To(corev1.PullAlways), - }, - SecurityContext: &corev1.SecurityContext{ - RunAsUser: ptr.To(int64(222)), - }, - Resources: &corev1.ResourceRequirements{ - Limits: corev1.ResourceList{"cpu": resource.MustParse("201m")}, - Requests: corev1.ResourceList{"cpu": resource.MustParse("203m")}, - }, - Bootstrap: &gw2_v1alpha1.SdsBootstrap{ - LogLevel: ptr.To("debug"), - }, - }, + SdsContainer: defaultSdsContainer(), PodTemplate: &gw2_v1alpha1.Pod{ ExtraAnnotations: map[string]string{ "pod-anno": "foo", diff --git a/projects/gateway2/deployer/values.go b/projects/gateway2/deployer/values.go index 7841effa807..2cb62967ef1 100644 --- a/projects/gateway2/deployer/values.go +++ b/projects/gateway2/deployer/values.go @@ -66,6 +66,9 @@ type helmGateway struct { // AWS values Aws *helmAws `json:"aws,omitempty"` + + // Gloo mTLS + GlooMtls *helmMtlsConfig `json:"glooMtls,omitempty" desc:"Config used to enable internal mtls authentication."` } // helmPort represents a Gateway Listener port @@ -162,3 +165,15 @@ type helmAws struct { StsClusterName *string `json:"stsClusterName,omitempty"` StsUri *string `json:"stsUri,omitempty"` } + +type helmTlsSecretData struct { + CaCert []byte `json:"caCert,omitempty"` + TlsCert []byte `json:"tlsCert,omitempty"` + TlsKey []byte `json:"tlsKey,omitempty"` +} + +type helmMtlsConfig struct { + Enabled *bool `json:"enabled,omitempty" desc:"Enables internal mtls authentication"` + RenderSecret *bool `json:"renderSecret,omitempty" desc:"If true, the deployer will render the mtls secrets. Used for creating the intial sceret resource for mointoring before the full config is available"` + TlsSecret *helmTlsSecretData `json:"tlsCert,omitempty" desc:"The tls cert and key for the gateway to use for mtls authentication"` +} diff --git a/projects/gateway2/helm/gloo-gateway/templates/gateway/proxy-deployment.yaml b/projects/gateway2/helm/gloo-gateway/templates/gateway/proxy-deployment.yaml index 0cabe3b5247..872629c5cd3 100644 --- a/projects/gateway2/helm/gloo-gateway/templates/gateway/proxy-deployment.yaml +++ b/projects/gateway2/helm/gloo-gateway/templates/gateway/proxy-deployment.yaml @@ -1,5 +1,24 @@ {{- $gateway := .Values.gateway }} {{- $statsConfig := $gateway.stats }} +{{- $glooMtls := dict -}} +{{- if $gateway.glooMtls }} + {{- $glooMtls = $gateway.glooMtls -}} +{{- end }} {{/* if $gateway.glooMtls.enabled */}} + +{{- if or $glooMtls.enabled $glooMtls.renderSecret }} +apiVersion: v1 +kind: Secret +metadata: + name: gloo-mtls-certs +data: + {{- if $glooMtls.tlsCert }} {{/* When initially rendering the template to get the list of GVKs to watch there won't be secret data */}} + ca.crt: {{ $glooMtls.tlsCert.caCert }} + tls.crt: {{ $glooMtls.tlsCert.tlsCert }} + tls.key: {{ $glooMtls.tlsCert.tlsKey }} + {{- end }} {{/* if $glooMtls.tlsCert */}} +type: kubernetes.io/tls +{{- end }} {{/* if $glooMtls.enabled */}} +--- apiVersion: apps/v1 kind: Deployment metadata: @@ -68,6 +87,11 @@ spec: volumeMounts: - mountPath: /etc/envoy name: envoy-config +{{- if $glooMtls.enabled }} + - mountPath: /etc/envoy/ssl + name: gloo-mtls-certs + readOnly: true +{{- end }} {{/* if $glooMtls.enabled*/}} env: - name: POD_NAME valueFrom: @@ -110,7 +134,7 @@ spec: resources: {{- toYaml $gateway.resources | nindent 10 }} {{- end }} {{/* if $gateway.resources */}} -{{- if $gateway.istio.enabled }} +{{- if or $glooMtls.enabled $gateway.istio.enabled }} - name: sds image: "{{ template "gloo-gateway.gateway.image" $gateway.sdsContainer.image }}" {{- if $gateway.sdsContainer.image.pullPolicy }} @@ -127,8 +151,14 @@ spec: fieldRef: apiVersion: v1 fieldPath: metadata.namespace +{{- if $gateway.istio.enabled }} - name: ISTIO_MTLS_SDS_ENABLED value: "true" +{{- end }} {{/* if $gateway.istio.enabled */}} +{{- if $glooMtls.enabled }} + - name: GLOO_MTLS_SDS_ENABLED + value: "true" +{{- end }} {{/* $glooMtls.enabled */}} - name: LOG_LEVEL value: {{ $gateway.sdsContainer.sdsBootstrap.logLevel }} ports: @@ -154,9 +184,16 @@ spec: terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: +{{- if $gateway.istio.enabled }} - mountPath: /etc/envoy name: envoy-config {{- end }} {{/* if $gateway.istio.enabled */}} +{{- if $glooMtls.enabled }} + - mountPath: /etc/envoy/ssl + name: gloo-mtls-certs + readOnly: true +{{- end }} {{/* if $glooMtls.enabled */}} +{{- end }} {{/* if or $glooMtls.enabled $gateway.istio.enabled */}} {{- if $gateway.istio.enabled }} - mountPath: /etc/istio-certs/ name: istio-certs @@ -316,6 +353,12 @@ spec: - configMap: name: {{ include "gloo-gateway.gateway.fullname" . }} name: envoy-config +{{- if $glooMtls.enabled }} + - name: gloo-mtls-certs + secret: + defaultMode: 420 + secretName: gloo-mtls-certs +{{- end }} {{/* if $glooMtls.enabled */}} {{- if (($gateway.aiExtension).enabled) }} - configMap: name: {{ include "gloo-gateway.gateway.fullname" . }}-ai-stats-config @@ -546,6 +589,33 @@ data: keepalive_time: 10 type: STRICT_DNS respect_dns_ttl: true +{{- if $glooMtls.enabled }} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_certificate_sds_secret_configs: + - name: server_cert + sds_config: + resource_api_version: V3 + api_config_source: + api_type: GRPC + transport_api_version: V3 + grpc_services: + - envoy_grpc: + cluster_name: gateway_proxy_sds + validation_context_sds_secret_config: + name: validation_context + sds_config: + resource_api_version: V3 + api_config_source: + api_type: GRPC + transport_api_version: V3 + grpc_services: + - envoy_grpc: + cluster_name: gateway_proxy_sds +{{- end }} {{/* if $glooMtls.enabled */}} - name: admin_port_cluster connect_timeout: 5.000s type: STATIC @@ -559,7 +629,7 @@ data: socket_address: address: 127.0.0.1 port_value: 19000 - {{- if $gateway.istio.enabled }} + {{- if or $glooMtls.enabled $gateway.istio.enabled }} - name: gateway_proxy_sds connect_timeout: 0.25s http2_protocol_options: {} @@ -572,7 +642,7 @@ data: socket_address: address: 127.0.0.1 port_value: 8234 - {{- end }} {{/* if $gateway.istio.enabled */}} + {{- end }} {{/* if or $glooMtls.enabled $gateway.istio.enabled */}} {{- if ($gateway.aws).enableServiceAccountCredentials }} - name: {{ $gateway.aws.stsClusterName }} connect_timeout: 5.000s diff --git a/projects/gateway2/setup/ggv2setup.go b/projects/gateway2/setup/ggv2setup.go index 93b39959e11..984ba264e90 100644 --- a/projects/gateway2/setup/ggv2setup.go +++ b/projects/gateway2/setup/ggv2setup.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "os" "sort" "strings" @@ -37,12 +38,14 @@ import ( k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + schema "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" ) var settingsGVR = glookubev1.SchemeGroupVersion.WithResource("settings") +var deploymentGVR = schema.GroupVersion{Group: "apps", Version: "v1"}.WithResource("deployments") func createKubeClient(restConfig *rest.Config) (istiokube.Client, error) { restCfg := istiokube.NewClientConfigForRestConfig(restConfig) @@ -76,6 +79,11 @@ func getInitialSettings(ctx context.Context, c istiokube.Client, nns types.Names return out } +// checkGlooMtlsEnabled checks if gloo mtls is enabled by looking at the gloo deployment and checking if the sds container is present +func checkGlooMtlsEnabled() bool { + return os.Getenv("GLOO_MTLS_SDS_ENABLED") == "true" +} + func StartGGv2(ctx context.Context, setupOpts *bootstrap.SetupOpts, uccBuilder krtcollections.UniquelyConnectedClientsBulider, @@ -137,6 +145,9 @@ func StartGGv2WithConfig(ctx context.Context, serviceClient := kclient.New[*corev1.Service](kubeClient) services := krt.WrapClient(serviceClient, krt.WithName("Services")) + logger.Info("checking if gloo mtls is enabled") + glooMtls := checkGlooMtlsEnabled() + logger.Info("creating reporter") kubeGwStatusReporter := NewGenericStatusReporter(kubeClient, defaults.KubeGatewayReporter) @@ -161,13 +172,15 @@ func StartGGv2WithConfig(ctx context.Context, InitialSettings: initialSettings, Settings: settingsSingle, // Dev flag may be useful for development purposes; not currently tied to any user-facing API - Dev: false, - Debugger: setupOpts.KrtDebugger, + Dev: false, + GlooMtlsEnabled: glooMtls, + Debugger: setupOpts.KrtDebugger, }) if err != nil { logger.Error("failed initializing controller: ", err) return err } + /// no collections after this point logger.Info("waiting for cache sync") diff --git a/projects/gateway2/wellknown/controller.go b/projects/gateway2/wellknown/controller.go index 6f2444d2c98..5d6ba54fb9f 100644 --- a/projects/gateway2/wellknown/controller.go +++ b/projects/gateway2/wellknown/controller.go @@ -20,4 +20,7 @@ const ( // DefaultGatewayParametersName is the name of the GatewayParameters which is attached by // parametersRef to the GatewayClass. DefaultGatewayParametersName = "gloo-gateway" + + // GlooMtlsCertName is the name of the TLS secret that contains the Gloo mTLS Certificates + GlooMtlsCertName = "gloo-mtls-certs" ) diff --git a/test/kubernetes/e2e/features/gloomtls/gloomtls_edge_suite.go b/test/kubernetes/e2e/features/gloomtls/gloomtls_edge_suite.go index db4c509b78a..b591b5f2f18 100644 --- a/test/kubernetes/e2e/features/gloomtls/gloomtls_edge_suite.go +++ b/test/kubernetes/e2e/features/gloomtls/gloomtls_edge_suite.go @@ -9,6 +9,7 @@ import ( "github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd/istio" "github.com/solo-io/gloo/test/gomega/matchers" testdefaults "github.com/solo-io/gloo/test/kubernetes/e2e/defaults" + "github.com/solo-io/gloo/test/kubernetes/e2e/tests/base" "github.com/stretchr/testify/suite" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -21,44 +22,22 @@ var _ e2e.NewSuiteFunc = NewGloomtlsEdgeGatewayApiTestingSuite // gloomtlsEdgeGatewayTestingSuite is the entire Suite of tests for the "Gloo mtls" cases type gloomtlsEdgeGatewayTestingSuite struct { - suite.Suite - - ctx context.Context - - // testInstallation contains all the metadata/utilities necessary to execute a series of tests - // against an installation of Gloo Gateway - testInstallation *e2e.TestInstallation + *base.BaseTestingSuite } func NewGloomtlsEdgeGatewayApiTestingSuite(ctx context.Context, testInst *e2e.TestInstallation) suite.TestingSuite { return &gloomtlsEdgeGatewayTestingSuite{ - ctx: ctx, - testInstallation: testInst, + base.NewBaseTestingSuite(ctx, testInst, edgeGatewaySetupSuite, nil), } } -func (s *gloomtlsEdgeGatewayTestingSuite) SetupSuite() { - err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, testdefaults.NginxPodManifest) - s.NoError(err, "can apply Nginx setup manifest") - err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, testdefaults.CurlPodManifest) - s.NoError(err, "can apply Curl setup manifest") - -} - -func (s *gloomtlsEdgeGatewayTestingSuite) TearDownSuite() { - err := s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, testdefaults.NginxPodManifest) - s.NoError(err, "can delete Nginx setup manifest") - err = s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, testdefaults.CurlPodManifest) - s.NoError(err, "can delete Curl setup manifest") -} - func (s *gloomtlsEdgeGatewayTestingSuite) TestRouteSecureRequestToUpstream() { s.T().Cleanup(func() { - err := s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, edgeRoutingResources, "-n", s.testInstallation.Metadata.InstallNamespace) + err := s.TestInstallation.Actions.Kubectl().DeleteFile(s.Ctx, edgeRoutingResources, "-n", s.TestInstallation.Metadata.InstallNamespace) s.NoError(err) }) - err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, edgeRoutingResources, "-n", s.testInstallation.Metadata.InstallNamespace) + err := s.TestInstallation.Actions.Kubectl().ApplyFile(s.Ctx, edgeRoutingResources, "-n", s.TestInstallation.Metadata.InstallNamespace) s.NoError(err) // Check sds container is present @@ -69,14 +48,14 @@ func (s *gloomtlsEdgeGatewayTestingSuite) TestRouteSecureRequestToUpstream() { matchers.PodMatches(matchers.ExpectedPod{ContainerName: istio.SDSContainerName}), ) - s.testInstallation.Assertions.EventuallyPodsMatches(s.ctx, s.testInstallation.Metadata.InstallNamespace, listOpts, matcher, time.Minute*2) + s.TestInstallation.Assertions.EventuallyPodsMatches(s.Ctx, s.TestInstallation.Metadata.InstallNamespace, listOpts, matcher, time.Minute*2) // Check curl works - s.testInstallation.Assertions.AssertEventualCurlResponse( - s.ctx, + s.TestInstallation.Assertions.AssertEventualCurlResponse( + s.Ctx, testdefaults.CurlPodExecOpt, []curl.Option{ - curl.WithHost(kubeutils.ServiceFQDN(metav1.ObjectMeta{Name: defaults.GatewayProxyName, Namespace: s.testInstallation.Metadata.InstallNamespace})), + curl.WithHost(kubeutils.ServiceFQDN(metav1.ObjectMeta{Name: defaults.GatewayProxyName, Namespace: s.TestInstallation.Metadata.InstallNamespace})), // The host header must match the domain in the VirtualService curl.WithHostHeader("example.com"), curl.WithPort(80), diff --git a/test/kubernetes/e2e/features/gloomtls/gloomtls_k8s_gateway_suite.go b/test/kubernetes/e2e/features/gloomtls/gloomtls_k8s_gateway_suite.go new file mode 100644 index 00000000000..6ff3eba206e --- /dev/null +++ b/test/kubernetes/e2e/features/gloomtls/gloomtls_k8s_gateway_suite.go @@ -0,0 +1,114 @@ +package gloomtls + +import ( + "context" + "encoding/json" + "path/filepath" + "time" + + "github.com/onsi/gomega" + "github.com/solo-io/gloo/pkg/utils/kubeutils" + "github.com/solo-io/gloo/pkg/utils/requestutils/curl" + "github.com/solo-io/gloo/projects/gateway2/wellknown" + "github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd/istio" + "github.com/solo-io/gloo/test/gomega/matchers" + "github.com/solo-io/gloo/test/kubernetes/e2e" + testdefaults "github.com/solo-io/gloo/test/kubernetes/e2e/defaults" + "github.com/solo-io/gloo/test/kubernetes/e2e/tests/base" + "github.com/solo-io/skv2/codegen/util" + "github.com/stretchr/testify/suite" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var _ e2e.NewSuiteFunc = NewGloomtlsK8sGatewayApiTestingSuite + +type gloomtlsK8sGatewayTestingSuite struct { + *base.BaseTestingSuite +} + +func NewGloomtlsK8sGatewayApiTestingSuite(ctx context.Context, testInst *e2e.TestInstallation) suite.TestingSuite { + return &gloomtlsK8sGatewayTestingSuite{ + base.NewBaseTestingSuite(ctx, testInst, base.SimpleTestCase{}, k8sGatewayTestCases), + } +} + +func (s *gloomtlsK8sGatewayTestingSuite) TestRouteSecureRequestToUpstream() { + // Check sds container is present + listOpts := metav1.ListOptions{ + LabelSelector: "gloo=kube-gateway", + } + matcher := gomega.And( + matchers.PodMatches(matchers.ExpectedPod{ContainerName: istio.SDSContainerName}), + ) + s.TestInstallation.Assertions.EventuallyPodsMatches(s.Ctx, "default", listOpts, matcher, time.Minute*2) + + s.ensureGlooAndProxyCertsMatch() + + // Check curl works + s.TestInstallation.Assertions.EventuallyRunningReplicas(s.Ctx, glooProxyObjectMeta, gomega.Equal(1)) + s.TestInstallation.Assertions.AssertEventualCurlResponse( + s.Ctx, + testdefaults.CurlPodExecOpt, + []curl.Option{ + curl.WithHost(kubeutils.ServiceFQDN(proxyService.ObjectMeta)), + curl.WithHostHeader("example.com"), + }, + expectedHealthyResponse) + + // Get the certs before the upgrade to ensure it was rotated + oldCerts := s.getMtlsCerts(s.TestInstallation.Metadata.InstallNamespace) + + // Run the certgen job manually instead of upgrading - this simulates the cronjob + s.rotateMtlsCerts() + + newCerts := s.getMtlsCerts(s.TestInstallation.Metadata.InstallNamespace) + s.NotEqual(oldCerts.Data, newCerts.Data) + + s.ensureGlooAndProxyCertsMatch() + + s.TestInstallation.Actions.Kubectl().ApplyFile(s.Ctx, filepath.Join(util.MustGetThisDir(), "testdata/hello-route.yaml")) + s.T().Cleanup(func() { + s.TestInstallation.Actions.Kubectl().DeleteFile(s.Ctx, filepath.Join(util.MustGetThisDir(), "testdata/hello-route.yaml")) + }) + + // Check curl works on the new route + s.TestInstallation.Assertions.EventuallyRunningReplicas(s.Ctx, glooProxyObjectMeta, gomega.Equal(1)) + s.TestInstallation.Assertions.AssertEventualCurlResponse( + s.Ctx, + testdefaults.CurlPodExecOpt, + []curl.Option{ + curl.WithHost(kubeutils.ServiceFQDN(proxyService.ObjectMeta)), + curl.WithHostHeader("hello.com"), + }, + expectedHealthyResponse) +} + +// ensureGlooAndProxyCertsMatch checks that the gloo Mtls certs that exist in the installation namespace and the proxy namespace are the same +func (s *gloomtlsK8sGatewayTestingSuite) ensureGlooAndProxyCertsMatch() { + glooSystemNsMtlsSecret := s.getMtlsCerts(s.TestInstallation.Metadata.InstallNamespace) + defaultNsMtlsSecret := s.getMtlsCerts("default") + s.TestInstallation.Assertions.Assert.Equal(defaultNsMtlsSecret.Data, glooSystemNsMtlsSecret.Data) +} + +func (s *gloomtlsK8sGatewayTestingSuite) getMtlsCerts(namespace string) *corev1.Secret { + secretString, _, err := s.TestInstallation.Actions.Kubectl().Get(s.Ctx, "secret", wellknown.GlooMtlsCertName, "-n", namespace, "-o", "json") + s.NoError(err) + + var mtlsSecret corev1.Secret + err = json.Unmarshal([]byte(secretString), &mtlsSecret) + s.NoError(err) + + return &mtlsSecret +} + +func (s *gloomtlsK8sGatewayTestingSuite) rotateMtlsCerts() { + // Delete the job if it still exists after completion. This ensures that the job will run and the certs rotated + s.TestInstallation.Actions.Kubectl().DeleteFile(s.Ctx, filepath.Join(util.MustGetThisDir(), "testdata/certgen.yaml"), "-n", s.TestInstallation.Metadata.InstallNamespace) + err := s.TestInstallation.Actions.Kubectl().ApplyFile(s.Ctx, filepath.Join(util.MustGetThisDir(), "testdata/certgen.yaml"), "-n", s.TestInstallation.Metadata.InstallNamespace) + s.NoError(err) + + // Wait until the job has completed and the certs have been rotated + s.TestInstallation.Actions.Kubectl().RunCommand(s.Ctx, "-n", s.TestInstallation.Metadata.InstallNamespace, "wait", "--for=condition=complete", "job", "gloo-mtls-certgen", "--timeout=600s") + +} diff --git a/test/kubernetes/e2e/features/gloomtls/testdata/certgen.yaml b/test/kubernetes/e2e/features/gloomtls/testdata/certgen.yaml new file mode 100644 index 00000000000..7c68855218c --- /dev/null +++ b/test/kubernetes/e2e/features/gloomtls/testdata/certgen.yaml @@ -0,0 +1,42 @@ +# Source: gloo/templates/19-gloo-mtls-certgen-job.yaml +apiVersion: batch/v1 +kind: Job +metadata: + labels: + app: gloo + gloo: gloo-mtls-certgen + name: gloo-mtls-certgen +spec: + ttlSecondsAfterFinished: 60 + template: + metadata: + labels: + gloo: gloo-mtls-certs + sidecar.istio.io/inject: "false" + spec: + serviceAccountName: certgen + restartPolicy: OnFailure + containers: + - image: quay.io/solo-io/certgen:1.0.0-ci1 + imagePullPolicy: IfNotPresent + name: certgen + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + runAsUser: 10101 + seccompProfile: + type: RuntimeDefault + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + args: + - "--secret-name=gloo-mtls-certs" + - "--svc-name=gloo" + - "--rotation-duration=65s" + - "--force-rotation=true" + diff --git a/test/kubernetes/e2e/features/gloomtls/testdata/hello-route.yaml b/test/kubernetes/e2e/features/gloomtls/testdata/hello-route.yaml new file mode 100644 index 00000000000..0a58a2220f4 --- /dev/null +++ b/test/kubernetes/e2e/features/gloomtls/testdata/hello-route.yaml @@ -0,0 +1,13 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: hello-route +spec: + parentRefs: + - name: gw + hostnames: + - "hello.com" + rules: + - backendRefs: + - name: example-svc + port: 8080 diff --git a/test/kubernetes/e2e/features/gloomtls/testdata/route-with-service.yaml b/test/kubernetes/e2e/features/gloomtls/testdata/route-with-service.yaml new file mode 100644 index 00000000000..63318dd348d --- /dev/null +++ b/test/kubernetes/e2e/features/gloomtls/testdata/route-with-service.yaml @@ -0,0 +1,27 @@ +kind: Gateway +apiVersion: gateway.networking.k8s.io/v1 +metadata: + name: gw +spec: + gatewayClassName: gloo-gateway + listeners: + - protocol: HTTP + port: 8080 + name: http + allowedRoutes: + namespaces: + from: Same +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: example-route +spec: + parentRefs: + - name: gw + hostnames: + - "example.com" + rules: + - backendRefs: + - name: example-svc + port: 8080 diff --git a/test/kubernetes/e2e/features/gloomtls/testdata/service-for-route.yaml b/test/kubernetes/e2e/features/gloomtls/testdata/service-for-route.yaml new file mode 100644 index 00000000000..8944dc7be68 --- /dev/null +++ b/test/kubernetes/e2e/features/gloomtls/testdata/service-for-route.yaml @@ -0,0 +1,26 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: example-svc +spec: + selector: + app.kubernetes.io/name: nginx + ports: + - protocol: TCP + port: 8080 + targetPort: http-web-svc +--- +apiVersion: v1 +kind: Pod +metadata: + name: nginx + labels: + app.kubernetes.io/name: nginx +spec: + containers: + - name: nginx + image: nginx:stable + ports: + - containerPort: 80 + name: http-web-svc diff --git a/test/kubernetes/e2e/features/gloomtls/types.go b/test/kubernetes/e2e/features/gloomtls/types.go index 4afd723c87f..7fb90f372aa 100644 --- a/test/kubernetes/e2e/features/gloomtls/types.go +++ b/test/kubernetes/e2e/features/gloomtls/types.go @@ -6,7 +6,13 @@ import ( . "github.com/onsi/gomega" testmatchers "github.com/solo-io/gloo/test/gomega/matchers" + "github.com/solo-io/gloo/test/kubernetes/e2e/defaults" + "github.com/solo-io/gloo/test/kubernetes/e2e/tests/base" "github.com/solo-io/skv2/codegen/util" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" ) var ( @@ -16,4 +22,29 @@ var ( } edgeRoutingResources = filepath.Join(util.MustGetThisDir(), "testdata", "edge_resources.yaml") + + edgeGatewaySetupSuite = base.SimpleTestCase{ + Manifests: []string{defaults.CurlPodManifest, defaults.NginxPodManifest}, + Resources: []client.Object{defaults.CurlPod, defaults.NginxPod}, + } + + // K8s Gateway tests + routeWithServiceManifest = filepath.Join(util.MustGetThisDir(), "testdata", "route-with-service.yaml") + serviceManifest = filepath.Join(util.MustGetThisDir(), "testdata", "service-for-route.yaml") + + glooProxyObjectMeta = metav1.ObjectMeta{ + Name: "gloo-proxy-gw", + Namespace: "default", + } + proxyDeployment = &appsv1.Deployment{ObjectMeta: glooProxyObjectMeta} + proxyService = &corev1.Service{ObjectMeta: glooProxyObjectMeta} + + k8sGatewayTestCases = map[string]*base.TestCase{ + "TestRouteSecureRequestToUpstream": { + SimpleTestCase: base.SimpleTestCase{ + Manifests: []string{defaults.CurlPodManifest, serviceManifest, routeWithServiceManifest}, + Resources: []client.Object{proxyDeployment, proxyService, defaults.CurlPod}, + }, + }, + } ) diff --git a/test/kubernetes/e2e/tests/gloomtls_k8s_gw_test.go b/test/kubernetes/e2e/tests/gloomtls_k8s_gw_test.go new file mode 100644 index 00000000000..68428fbdf18 --- /dev/null +++ b/test/kubernetes/e2e/tests/gloomtls_k8s_gw_test.go @@ -0,0 +1,54 @@ +package tests_test + +import ( + "context" + "os" + "testing" + "time" + + "github.com/solo-io/gloo/pkg/utils/envutils" + "github.com/solo-io/gloo/test/kubernetes/e2e" + . "github.com/solo-io/gloo/test/kubernetes/e2e/tests" + "github.com/solo-io/gloo/test/kubernetes/testutils/gloogateway" + "github.com/solo-io/gloo/test/testutils" +) + +// TestGloomtlsGatewayK8sGateway is the function which executes a series of tests against a given installation where +// the k8s Gateway controller is enabled and gloomtls is enabled +func TestGloomtlsGatewayK8sGateway(t *testing.T) { + ctx := context.Background() + installNs, nsEnvPredefined := envutils.LookupOrDefault(testutils.InstallNamespace, "gloo-gateway-k8s-gw-test") + testInstallation := e2e.CreateTestInstallation( + t, + &gloogateway.Context{ + InstallNamespace: installNs, + ProfileValuesManifestFile: e2e.KubernetesGatewayProfilePath, + ValuesManifestFile: e2e.ManifestPath("gloomtls-edge-gateway-test-helm.yaml"), + }, + ) + + testHelper := e2e.MustTestHelper(ctx, testInstallation) + + // Set the env to the install namespace if it is not already set + if !nsEnvPredefined { + os.Setenv(testutils.InstallNamespace, installNs) + } + + // We register the cleanup function _before_ we actually perform the installation. + // This allows us to uninstall Gloo Gateway, in case the original installation only completed partially + t.Cleanup(func() { + if !nsEnvPredefined { + os.Unsetenv(testutils.InstallNamespace) + } + if t.Failed() { + testInstallation.PreFailHandler(ctx) + } + + testInstallation.UninstallGlooGatewayWithTestHelper(ctx, testHelper) + }) + + // Install Gloo Gateway with only Gloo Edge Gateway APIs enabled + testInstallation.InstallGlooGatewayWithTestHelper(ctx, testHelper, 5*time.Minute) + + GloomtlsK8sGwSuiteRunner().Run(ctx, t, testInstallation) +} diff --git a/test/kubernetes/e2e/tests/gloomtls_k8s_gw_tests.go b/test/kubernetes/e2e/tests/gloomtls_k8s_gw_tests.go new file mode 100644 index 00000000000..35d96948bda --- /dev/null +++ b/test/kubernetes/e2e/tests/gloomtls_k8s_gw_tests.go @@ -0,0 +1,12 @@ +package tests + +import ( + "github.com/solo-io/gloo/test/kubernetes/e2e" + "github.com/solo-io/gloo/test/kubernetes/e2e/features/gloomtls" +) + +func GloomtlsK8sGwSuiteRunner() e2e.SuiteRunner { + gloomtlsEdgeGwSuiteRunner := e2e.NewSuiteRunner(false) + gloomtlsEdgeGwSuiteRunner.Register("Gloomtls", gloomtls.NewGloomtlsK8sGatewayApiTestingSuite) + return gloomtlsEdgeGwSuiteRunner +} diff --git a/test/kubernetes/e2e/tests/manifests/gloomtls-edge-gateway-test-helm.yaml b/test/kubernetes/e2e/tests/manifests/gloomtls-edge-gateway-test-helm.yaml index 2ebde2f244d..93b7d976d1a 100644 --- a/test/kubernetes/e2e/tests/manifests/gloomtls-edge-gateway-test-helm.yaml +++ b/test/kubernetes/e2e/tests/manifests/gloomtls-edge-gateway-test-helm.yaml @@ -1,3 +1,6 @@ +gateway: + certGenJob: + forceRotation: true global: glooMtls: enabled: true diff --git a/tilt-settings.yaml b/tilt-settings.yaml index 52d8466c99c..8fe1e55fba8 100644 --- a/tilt-settings.yaml +++ b/tilt-settings.yaml @@ -2,12 +2,12 @@ helm_installation_name: gloo-oss helm_values_files: - ./test/kubernetes/e2e/tests/manifests/common-recommendations.yaml -# - ./test/kubernetes/e2e/tests/manifests/profiles/kubernetes-gateway.yaml +- ./test/kubernetes/e2e/tests/manifests/profiles/kubernetes-gateway.yaml helm_installation_namespace: gloo-system enabled_providers: - gloo - - gateway-proxy + # - gateway-proxy metal_lb: false @@ -17,6 +17,7 @@ providers: image: quay.io/solo-io/gloo live_reload_deps: - projects/gloo + - projects/gateway2 label: gloo build_binary: GCFLAGS='all="-N -l"' make -B gloo binary_name: gloo-linux-$ARCH