Add support for TLS for xDS with Gloo Gateway (#10582)

Co-authored-by: David Jumani <[email protected]>
Co-authored-by: Jenny Shu <[email protected]>

.github/workflows/pr-kubernetes-tests.yaml

@@ -85,10 +85,10 @@ jobs:
           go-test-args: '-v -timeout=30m'
           go-test-run-regex: '^TestDiscoveryWatchlabels$$|^TestK8sGatewayNoValidation$$|^TestHelm$$|^TestHelmSettings$$|^TestK8sGatewayAws$$|^TestK8sGateway$$/^HTTPRouteServices$$|^TestK8sGateway$$/^TCPRouteServices$$|^TestZeroDowntimeRollout$$'
 
-        # Dec 4, 2024: 13 minutes
+        # Dec 4, 2024: 16 minutes
         - cluster-name: 'cluster-seven'
           go-test-args: '-v -timeout=25m'
-          go-test-run-regex: '^TestK8sGateway$$/^CRDCategories$$|^TestK8sGateway$$/^Metrics$$|^TestGloomtlsGatewayEdgeGateway$$|^TestWatchNamespaceSelector$$'
+          go-test-run-regex: '^TestK8sGateway$$/^CRDCategories$$|^TestK8sGateway$$/^Metrics$$|^TestGloomtlsGatewayEdgeGateway$$|^TestGloomtlsGatewayK8sGateway$$|^TestWatchNamespaceSelector$$'
 
         # In our PR tests, we run the suite of tests using the upper ends of versions that we claim to support
         # The versions should mirror: https://docs.solo.io/gloo-edge/latest/reference/support/

changelog/v1.19.0-beta5/k8s-gw-mtls-deployer.yaml

@@ -0,0 +1,6 @@
+changelog:
+  - type: NEW_FEATURE
+    issueLink: https://github.com/solo-io/solo-projects/issues/6210
+    resolvesIssue: false
+    description: >-
+      Add support for xDS over mTLS for communication between the Gloo pod and the Kubernetes Gateway proxies. This can be enabled by setting the 'global.glooMtls.enabled' helm value to true.

install/helm/gloo/generate/values.go

@@ -340,7 +340,6 @@ type GatewayParameters struct {
 	AIExtension     *GatewayParamsAIExtension  `json:"aiExtension,omitempty" desc:"Config used to manage the Gloo Gateway AI extension."`
 	FloatingUserId  *bool                      `json:"floatingUserId,omitempty" desc:"If true, allows the cluster to dynamically assign a user ID for the processes running in the container. Default is false."`
 	PodTemplate     *GatewayParamsPodTemplate  `json:"podTemplate,omitempty" desc:"The template used to generate the gatewayParams pod"`
-	// TODO(npolshak): Add support for GlooMtls
 }
 
 // GatewayProxyPodTemplate contains the Helm API available to configure the PodTemplate on the gateway Deployment

install/helm/gloo/templates/1-gloo-deployment.yaml

@@ -273,6 +273,10 @@ spec:
           - name: HEADER_SECRET_REF_NS_MATCHES_US
             value: "true"
         {{- end}}
+        {{- if .Values.global.glooMtls.enabled }}
+          - name: GLOO_MTLS_SDS_ENABLED
+            value: "true"
+        {{- end }}
 {{- if not .Values.global.glooMtls.enabled }}
         readinessProbe:
           tcpSocket:

install/helm/gloo/templates/44-rbac.yaml

@@ -22,9 +22,14 @@ rules:
   - services
   - pods
   - nodes
-  - secrets
   - namespaces
   verbs: ["get", "list", "watch"]
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  {{/* This is needed as the gateway deployer would need to create / patch the mtls certs if enabled */}}
+  verbs: ["get", "list", "watch", "create", "patch"]
 - apiGroups:
   - "discovery.k8s.io"
   resources:

projects/gateway2/controller/controller.go

@@ -188,6 +188,11 @@ func (c *controllerBuilder) addHttpLisOptIndexes(ctx context.Context) error {
 	})
 }
 
+func (c *controllerBuilder) shouldWatchSecrets() bool {
+	// watch for secrets if mtls is enabled
+	return c.cfg.ControlPlane.GlooMtlsEnabled
+}
+
 func (c *controllerBuilder) watchGw(ctx context.Context) error {
 	// setup a deployer
 	log := log.FromContext(ctx)
@@ -224,8 +229,31 @@ func (c *controllerBuilder) watchGw(ctx context.Context) error {
 			),
 		))
 
-	// watch for changes in GatewayParameters
 	cli := c.cfg.Mgr.GetClient()
+
+	// watch for secrets if needed
+	if c.shouldWatchSecrets() {
+		buildr.Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(
+			func(ctx context.Context, obj client.Object) []reconcile.Request {
+				var reqs []reconcile.Request
+				if obj.GetName() == wellknown.GlooMtlsCertName && obj.GetNamespace() == c.cfg.ControlPlane.Namespace {
+					var gwList apiv1.GatewayList
+					err := cli.List(ctx, &gwList, client.InNamespace(corev1.NamespaceAll))
+					if err != nil {
+						log.Error(err, "could not list Gateways", "namespace", corev1.NamespaceAll)
+						return reqs
+					}
+
+					for _, gw := range gwList.Items {
+						reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKey{Namespace: gw.Namespace, Name: gw.Name}})
+					}
+					return reqs
+				}
+				return reqs
+			}))
+	}
+
+	// watch for changes in GatewayParameters
 	buildr.Watches(&v1alpha1.GatewayParameters{}, handler.EnqueueRequestsFromMapFunc(
 		func(ctx context.Context, obj client.Object) []reconcile.Request {
 			gwpName := obj.GetName()

projects/gateway2/controller/start.go

@@ -8,6 +8,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/config"
 
 	glooschemes "github.com/solo-io/gloo/pkg/schemes"
+	"github.com/solo-io/gloo/pkg/utils/namespaces"
 	"github.com/solo-io/go-utils/contextutils"
 
 	"k8s.io/apimachinery/pkg/util/sets"
@@ -77,6 +78,8 @@ type StartConfig struct {
 	InitialSettings *glookubev1.Settings
 	Settings        krt.Singleton[glookubev1.Settings]
 
+	GlooMtlsEnabled bool
+
 	Debugger *krt.DebugHandler
 }
 
@@ -221,7 +224,7 @@ func (c *ControllerBuilder) Start(ctx context.Context) error {
 		return ctx.Err()
 	}
 
-	logger.Infow("got xds address for deployer", uzap.String("xds_host", xdsHost), uzap.Int32("xds_port", xdsPort))
+	logger.Infow("got xds address for deployer", uzap.String("xds_host", xdsHost), uzap.Int32("xds_port", xdsPort), uzap.Any("glooMtlsEnabled", c.cfg.GlooMtlsEnabled))
 
 	integrationEnabled := c.cfg.InitialSettings.Spec.GetGloo().GetIstioOptions().GetEnableIntegration().GetValue()
 
@@ -255,8 +258,10 @@ func (c *ControllerBuilder) Start(ctx context.Context) error {
 		ControllerName: wellknown.GatewayControllerName,
 		AutoProvision:  AutoProvision,
 		ControlPlane: deployer.ControlPlaneInfo{
-			XdsHost: xdsHost,
-			XdsPort: xdsPort,
+			XdsHost:         xdsHost,
+			XdsPort:         xdsPort,
+			GlooMtlsEnabled: c.cfg.GlooMtlsEnabled,
+			Namespace:       namespaces.GetPodNamespace(),
 		},
 		// TODO pass in the settings so that the deloyer can register to it for changes.
 		IstioIntegrationEnabled: integrationEnabled,

projects/gateway2/deployer/deployer.go

@@ -25,6 +25,7 @@ import (
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/yaml"
 	"k8s.io/utils/ptr"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -53,6 +54,11 @@ type Deployer struct {
 type ControlPlaneInfo struct {
 	XdsHost string
 	XdsPort int32
+	// The data in this struct is static, so is a good place to keep track of if mtls is enabled
+	// and a bad place to store the actual mtls secret data
+	GlooMtlsEnabled bool
+	// We could lookup the pod namespace from the env, but it's cleaner to pass it in
+	Namespace string
 }
 
 type AwsInfo struct {
@@ -103,17 +109,21 @@ func (d *Deployer) GetGvksToWatch(ctx context.Context) ([]schema.GroupVersionKin
 	//   as we only care about the GVKs of the rendered resources)
 	// - the minimal values that render all the proxy resources (HPA is not included because it's not
 	//   fully integrated/working at the moment)
+	// - a flag to indicate whether mtls is enabled, so we can render the secret if needed
 	//
 	// Note: another option is to hardcode the GVKs here, but rendering the helm chart is a
 	// _slightly_ more dynamic way of getting the GVKs. It isn't a perfect solution since if
 	// we add more resources to the helm chart that are gated by a flag, we may forget to
 	// update the values here to enable them.
+	// Currently the only resource that is gated by a flag is the mtls secret.
+
 	emptyGw := &api.Gateway{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "default",
 			Namespace: "default",
 		},
 	}
+
 	// TODO(Law): these must be set explicitly as we don't have defaults for them
 	// and the internal template isn't robust enough.
 	// This should be empty eventually -- the template must be resilient against nil-pointers
@@ -124,13 +134,20 @@ func (d *Deployer) GetGvksToWatch(ctx context.Context) ([]schema.GroupVersionKin
 				"enabled": false,
 			},
 			"image": map[string]any{},
+			// Render the secret based on the mtls flag so we can watch it.
+			// This is an exception to the "TODO" above as this is not protection against nil-pointers,
+			// it is determining which resources to render based on ControlPlane configuration.
+			"glooMtls": map[string]any{
+				"renderSecret": d.inputs.ControlPlane.GlooMtlsEnabled,
+			},
 		},
 	}
 
 	objs, err := d.renderChartToObjects(emptyGw, vals)
 	if err != nil {
 		return nil, err
 	}
+
 	var ret []schema.GroupVersionKind
 	for _, obj := range objs {
 		gvk := obj.GetObjectKind().GroupVersionKind()
@@ -255,7 +272,7 @@ func (d *Deployer) getGatewayClassFromGateway(ctx context.Context, gw *api.Gatew
 	return gwc, nil
 }
 
-func (d *Deployer) getValues(gw *api.Gateway, gwParam *v1alpha1.GatewayParameters) (*helmConfig, error) {
+func (d *Deployer) getValues(ctx context.Context, gw *api.Gateway, gwParam *v1alpha1.GatewayParameters) (*helmConfig, error) {
 	// construct the default values
 	vals := &helmConfig{
 		Gateway: &helmGateway{
@@ -361,9 +378,61 @@ func (d *Deployer) getValues(gw *api.Gateway, gwParam *v1alpha1.GatewayParameter
 
 	gateway.Stats = getStatsValues(statsConfig)
 
+	// mtls values
+	gateway.GlooMtls, err = d.getHelmMtlsConfig(ctx)
+	if err != nil {
+		return nil, err
+	}
+
 	return vals, nil
 }
 
+func (d *Deployer) getHelmMtlsConfig(ctx context.Context) (*helmMtlsConfig, error) {
+
+	if !d.inputs.ControlPlane.GlooMtlsEnabled {
+		return &helmMtlsConfig{
+			Enabled: ptr.To(false),
+		}, nil
+	}
+
+	helmTls, err := d.getHelmTlsSecretData(ctx)
+
+	if err != nil {
+		return nil, err
+	}
+
+	return &helmMtlsConfig{
+		Enabled:   ptr.To(true),
+		TlsSecret: helmTls,
+	}, nil
+}
+
+// getHelmTlsSecretData builds a helmTls object built from the gloo-mtls-certs secret data, which it fetches
+// This function does not check if mtls is enabled, and a missing secret will return an error via getGlooMtlsCertsSecret
+func (d *Deployer) getHelmTlsSecretData(ctx context.Context) (*helmTlsSecretData, error) {
+
+	mtlsSecret := &corev1.Secret{}
+	mtlsSecretNns := types.NamespacedName{
+		Name:      wellknown.GlooMtlsCertName,
+		Namespace: d.inputs.ControlPlane.Namespace,
+	}
+	err := d.cli.Get(ctx, mtlsSecretNns, mtlsSecret)
+
+	if err != nil {
+		return nil, eris.Wrap(err, "failed to get gloo mtls secret")
+	}
+
+	if mtlsSecret.Type != corev1.SecretTypeTLS {
+		return nil, eris.New(fmt.Sprintf("unexpected secret type, expected %s and got %s", corev1.SecretTypeTLS, mtlsSecret.Type))
+	}
+
+	return &helmTlsSecretData{
+		TlsCert: mtlsSecret.Data[corev1.TLSCertKey],
+		TlsKey:  mtlsSecret.Data[corev1.TLSPrivateKeyKey],
+		CaCert:  mtlsSecret.Data[corev1.ServiceAccountRootCAKey],
+	}, nil
+}
+
 // Render relies on a `helm install` to render the Chart with the injected values
 // It returns the list of Objects that are rendered, and an optional error if rendering failed,
 // or converting the rendered manifests to objects failed.
@@ -392,6 +461,7 @@ func (d *Deployer) Render(name, ns string, vals map[string]any) ([]client.Object
 	if err != nil {
 		return nil, fmt.Errorf("failed to convert helm manifest yaml to objects for gateway %s.%s: %w", ns, name, err)
 	}
+
 	return objs, nil
 }
 
@@ -405,6 +475,8 @@ func (d *Deployer) Render(name, ns string, vals map[string]any) ([]client.Object
 //
 // * returns the objects to be deployed by the caller
 func (d *Deployer) GetObjsToDeploy(ctx context.Context, gw *api.Gateway) ([]client.Object, error) {
+	logger := log.FromContext(ctx)
+
 	gwParam, err := d.getGatewayParametersForGateway(ctx, gw)
 	if err != nil {
 		return nil, err
@@ -414,9 +486,7 @@ func (d *Deployer) GetObjsToDeploy(ctx context.Context, gw *api.Gateway) ([]clie
 		return nil, nil
 	}
 
-	logger := log.FromContext(ctx)
-
-	vals, err := d.getValues(gw, gwParam)
+	vals, err := d.getValues(ctx, gw, gwParam)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get values to render objects for gateway %s.%s: %w", gw.GetNamespace(), gw.GetName(), err)
 	}