TLS between Ingress and activator

.github/workflows/kind-e2e.yaml

@@ -192,13 +192,17 @@ jobs:
       run: |
         source ./test/e2e-common.sh
 
+        bash ./test/generate-cert.sh
+
         KIND=1
         INGRESS_CLASS="${{ matrix.kingress }}.ingress.networking.knative.dev"
         CLUSTER_DOMAIN="${{ matrix.cluster-suffix }}"
 
         knative_setup
         test_setup
 
+        kubectl apply -f ./test/config/tls
+
         echo "INGRESS_CLASS=$INGRESS_CLASS" >> $GITHUB_ENV
         echo "CLUSTER_DOMAIN=$CLUSTER_DOMAIN" >> $GITHUB_ENV
         echo "SYSTEM_NAMESPACE=$SYSTEM_NAMESPACE" >> $GITHUB_ENV

cmd/activator/main.go

@@ -68,6 +68,10 @@ const (
 
 	// The port on which autoscaler WebSocket server listens.
 	autoscalerPort = ":8080"
+
+	certDirectory = "/var/lib/knative/certs"
+	certPath      = certDirectory + "/tls.crt"
+	keyPath       = certDirectory + "/tls.key"
 )
 
 type config struct {
@@ -241,6 +245,22 @@ func main() {
 		}(name, server)
 	}
 
+	// TODO: Use configmap to enable tls mode.
+	if true {
+		tlsServers := map[string]*http.Server{
+			"https": pkgnet.NewServer(":"+strconv.Itoa(networking.BackendHTTPSPort), ah),
+		}
+
+		for name, server := range tlsServers {
+			go func(name string, s *http.Server) {
+				// Don't forward ErrServerClosed as that indicates we're already shutting down.
+				if err := s.ListenAndServeTLS(certPath, keyPath); err != nil && !errors.Is(err, http.ErrServerClosed) {
+					errCh <- fmt.Errorf("%s server failed: %w", name, err)
+				}
+			}(name, server)
+		}
+	}
+
 	// Wait for the signal to drain.
 	select {
 	case <-sigCtx.Done():

config/core/deployments/activator.yaml

@@ -54,6 +54,11 @@ spec:
             cpu: 1000m
             memory: 600Mi
 
+        volumeMounts:
+        - name: server-certs
+          mountPath: /var/lib/knative/certs
+          readOnly: true
+
         env:
         # Run Activator with GC collection when newly generated memory is 500%.
         - name: GOGC
@@ -123,6 +128,11 @@ spec:
       # connections.
       terminationGracePeriodSeconds: 600
 
+      volumes:
+      - name: server-certs
+        secret:
+          secretName: server-certs
+
 ---
 apiVersion: v1
 kind: Service
@@ -148,6 +158,9 @@ spec:
   - name: http
     port: 80
     targetPort: 8012
+  - name: https
+    port: 443
+    targetPort: 8112
   - name: http2
     port: 81
     targetPort: 8013

pkg/networking/constants.go

@@ -26,6 +26,9 @@ const (
 	// BackendHTTP2Port is the backend, i.e. `targetPort` that we setup for HTTP/2 services.
 	BackendHTTP2Port = 8013
 
+	// BackendHTTPSPort is the backend. i.e. `targetPort` that we setup for HTTPS services.
+	BackendHTTPSPort = 8112
+
 	// QueueAdminPort specifies the port number for
 	// health check and lifecycle hooks for queue-proxy.
 	QueueAdminPort = 8022

pkg/reconciler/route/resources/ingress.go

@@ -136,6 +136,11 @@ func makeIngressSpec(
 	rules := make([]netv1alpha1.IngressRule, 0, len(names))
 
 	featuresConfig := config.FromContextOrDefaults(ctx).Features
+	// Use ConfigMap to enable internalTLS.
+	internalTLS := false
+	if true {
+		internalTLS = true
+	}
 
 	for _, name := range names {
 		visibilities := []netv1alpha1.IngressVisibility{netv1alpha1.IngressVisibilityClusterLocal}
@@ -149,7 +154,7 @@ func makeIngressSpec(
 				return netv1alpha1.IngressSpec{}, err
 			}
 			rule := makeIngressRule(domains, r.Namespace,
-				visibility, tc.Targets[name], ro.RolloutsByTag(name))
+				visibility, tc.Targets[name], ro.RolloutsByTag(name), internalTLS)
 			if featuresConfig.TagHeaderBasedRouting == apicfg.Enabled {
 				if rule.HTTP.Paths[0].AppendHeaders == nil {
 					rule.HTTP.Paths[0].AppendHeaders = make(map[string]string, 1)
@@ -171,7 +176,7 @@ func makeIngressSpec(
 					// Since names are sorted `DefaultTarget == ""` is the first one,
 					// so just pass the subslice.
 					rule.HTTP.Paths = append(
-						makeTagBasedRoutingIngressPaths(r.Namespace, tc, ro, names[1:]), rule.HTTP.Paths...)
+						makeTagBasedRoutingIngressPaths(r.Namespace, tc, ro, internalTLS, names[1:]), rule.HTTP.Paths...)
 				} else {
 					// If a request is routed by a tag-attached hostname instead of the tag header,
 					// the request may not have the tag header "Knative-Serving-Tag",
@@ -290,24 +295,25 @@ func MakeACMEIngressPaths(acmeChallenges []netv1alpha1.HTTP01Challenge, domains
 func makeIngressRule(domains []string, ns string,
 	visibility netv1alpha1.IngressVisibility,
 	targets traffic.RevisionTargets,
-	roCfgs []*traffic.ConfigurationRollout) netv1alpha1.IngressRule {
+	roCfgs []*traffic.ConfigurationRollout,
+	internalTLS bool) netv1alpha1.IngressRule {
 	return netv1alpha1.IngressRule{
 		Hosts:      domains,
 		Visibility: visibility,
 		HTTP: &netv1alpha1.HTTPIngressRuleValue{
 			Paths: []netv1alpha1.HTTPIngressPath{
-				*makeBaseIngressPath(ns, targets, roCfgs),
+				*makeBaseIngressPath(ns, targets, roCfgs, internalTLS),
 			},
 		},
 	}
 }
 
 // `names` must not include `""` — the DefaultTarget.
-func makeTagBasedRoutingIngressPaths(ns string, tc *traffic.Config, ro *traffic.Rollout, names []string) []netv1alpha1.HTTPIngressPath {
+func makeTagBasedRoutingIngressPaths(ns string, tc *traffic.Config, ro *traffic.Rollout, internalTLS bool, names []string) []netv1alpha1.HTTPIngressPath {
 	paths := make([]netv1alpha1.HTTPIngressPath, 0, len(names))
 
 	for _, name := range names {
-		path := makeBaseIngressPath(ns, tc.Targets[name], ro.RolloutsByTag(name))
+		path := makeBaseIngressPath(ns, tc.Targets[name], ro.RolloutsByTag(name), internalTLS)
 		path.Headers = map[string]netv1alpha1.HeaderMatch{network.TagHeaderName: {Exact: name}}
 		paths = append(paths, *path)
 	}
@@ -327,7 +333,7 @@ func rolloutConfig(cfgName string, ros []*traffic.ConfigurationRollout) *traffic
 }
 
 func makeBaseIngressPath(ns string, targets traffic.RevisionTargets,
-	roCfgs []*traffic.ConfigurationRollout) *netv1alpha1.HTTPIngressPath {
+	roCfgs []*traffic.ConfigurationRollout, internalTLS bool) *netv1alpha1.HTTPIngressPath {
 	// Optimistically allocate |targets| elements.
 	splits := make([]netv1alpha1.IngressBackendSplit, 0, len(targets))
 	for _, t := range targets {
@@ -339,6 +345,10 @@ func makeBaseIngressPath(ns string, targets traffic.RevisionTargets,
 		if t.LatestRevision != nil && *t.LatestRevision {
 			cfg = rolloutConfig(t.ConfigurationName, roCfgs)
 		}
+		servicePort := intstr.FromInt(networking.ServicePort(t.Protocol))
+		if internalTLS {
+			servicePort = intstr.FromInt(443)
+		}
 		if cfg == nil || len(cfg.Revisions) < 2 {
 			// No rollout in progress.
 			splits = append(splits, netv1alpha1.IngressBackendSplit{
@@ -347,7 +357,7 @@ func makeBaseIngressPath(ns string, targets traffic.RevisionTargets,
 					ServiceName:      t.RevisionName,
 					// Port on the public service must match port on the activator.
 					// Otherwise, the serverless services can't guarantee seamless positive handoff.
-					ServicePort: intstr.FromInt(networking.ServicePort(t.Protocol)),
+					ServicePort: servicePort,
 				},
 				Percent: int(*t.Percent),
 				AppendHeaders: map[string]string{
@@ -364,7 +374,7 @@ func makeBaseIngressPath(ns string, targets traffic.RevisionTargets,
 						ServiceName:      rev.RevisionName,
 						// Port on the public service must match port on the activator.
 						// Otherwise, the serverless services can't guarantee seamless positive handoff.
-						ServicePort: intstr.FromInt(networking.ServicePort(t.Protocol)),
+						ServicePort: servicePort,
 					},
 					Percent: rev.Percent,
 					AppendHeaders: map[string]string{

pkg/reconciler/route/resources/ingress_test.go

@@ -866,8 +866,9 @@ func TestMakeIngressRuleVanilla(t *testing.T) {
 		},
 	}
 	ro := tc.BuildRollout()
+	internalTLS := false
 	rule := makeIngressRule(domains, ns,
-		netv1alpha1.IngressVisibilityExternalIP, targets, ro.RolloutsByTag(traffic.DefaultTarget))
+		netv1alpha1.IngressVisibilityExternalIP, targets, ro.RolloutsByTag(traffic.DefaultTarget), internalTLS)
 	expected := netv1alpha1.IngressRule{
 		Hosts: []string{
 			"a.com",
@@ -919,8 +920,9 @@ func TestMakeIngressRuleZeroPercentTarget(t *testing.T) {
 		},
 	}
 	ro := tc.BuildRollout()
+	internalTLS := false
 	rule := makeIngressRule(domains, ns,
-		netv1alpha1.IngressVisibilityExternalIP, targets, ro.RolloutsByTag(traffic.DefaultTarget))
+		netv1alpha1.IngressVisibilityExternalIP, targets, ro.RolloutsByTag(traffic.DefaultTarget), internalTLS)
 	expected := netv1alpha1.IngressRule{
 		Hosts: []string{"test.org"},
 		HTTP: &netv1alpha1.HTTPIngressRuleValue{
@@ -968,9 +970,10 @@ func TestMakeIngressRuleTwoTargets(t *testing.T) {
 		},
 	}
 	ro := tc.BuildRollout()
+	internalTLS := false
 	domains := []string{"test.org"}
 	rule := makeIngressRule(domains, ns, netv1alpha1.IngressVisibilityExternalIP,
-		targets, ro.RolloutsByTag("a-tag"))
+		targets, ro.RolloutsByTag("a-tag"), internalTLS)
 	expected := netv1alpha1.IngressRule{
 		Hosts: []string{"test.org"},
 		HTTP: &netv1alpha1.HTTPIngressRuleValue{

pkg/reconciler/serverlessservice/resources/services.go

@@ -53,16 +53,32 @@ func MakePublicService(sks *v1alpha1.ServerlessService) *corev1.Service {
 			OwnerReferences: []metav1.OwnerReference{*kmeta.NewControllerRef(sks)},
 		},
 		Spec: corev1.ServiceSpec{
-			Ports: []corev1.ServicePort{{
-				Name:       pkgnet.ServicePortName(sks.Spec.ProtocolType),
-				Protocol:   corev1.ProtocolTCP,
-				Port:       int32(pkgnet.ServicePort(sks.Spec.ProtocolType)),
-				TargetPort: targetPort(sks),
-			}},
+			Ports: makePublicServicePorts(sks),
 		},
 	}
 }
 
+func makePublicServicePorts(sks *v1alpha1.ServerlessService) []corev1.ServicePort {
+	ports := []corev1.ServicePort{{
+		Name:       pkgnet.ServicePortName(sks.Spec.ProtocolType),
+		Protocol:   corev1.ProtocolTCP,
+		Port:       int32(pkgnet.ServicePort(sks.Spec.ProtocolType)),
+		TargetPort: targetPort(sks),
+	}}
+
+	// TODO: Use annotation or sks.spec whether add HTTPS port or not.
+	if true {
+		p := corev1.ServicePort{
+			Name:       pkgnet.ServicePortNameHTTPS,
+			Protocol:   corev1.ProtocolTCP,
+			Port:       pkgnet.ServiceHTTPSPort,
+			TargetPort: intstr.FromInt(networking.BackendHTTPSPort),
+		}
+		ports = append(ports, p)
+	}
+	return ports
+}
+
 // MakePublicEndpoints constructs a K8s Endpoints that is not backed a selector
 // and will be manually reconciled by the SKS controller.
 func MakePublicEndpoints(sks *v1alpha1.ServerlessService, src *corev1.Endpoints) *corev1.Endpoints {
@@ -99,11 +115,20 @@ func filterSubsetPorts(targetPort int32, subsets []corev1.EndpointSubset) []core
 	ret := make([]corev1.EndpointSubset, len(subsets))
 	for i, sss := range subsets {
 		sst := sss.DeepCopy()
+		ssts := sss.DeepCopy()
 		// Find the port we care about and remove all others.
 		for j, p := range sst.Ports {
 			if p.Port == targetPort {
 				sst.Ports = sst.Ports[j : j+1]
-				break
+			}
+		}
+
+		// TODO: Use annotation or configmap to add HTTPS port.
+		if true {
+			for j, p := range ssts.Ports {
+				if p.Port == networking.BackendHTTPSPort {
+					sst.Ports = append(sst.Ports, ssts.Ports[j:j+1]...)
+				}
 			}
 		}
 		ret[i] = *sst
@@ -134,6 +159,12 @@ func MakePrivateService(sks *v1alpha1.ServerlessService, selector map[string]str
 				// This one is matching the public one, since this is the
 				// port queue-proxy listens on.
 				TargetPort: targetPort(sks),
+			}, {
+				// TODO: Add https port only when tls mode is enabled?
+				Name:       pkgnet.ServicePortNameHTTPS,
+				Protocol:   corev1.ProtocolTCP,
+				Port:       pkgnet.ServiceHTTPSPort,
+				TargetPort: intstr.FromInt(networking.BackendHTTPSPort),
 			}, {
 				Name:       servingv1.AutoscalingQueueMetricsPortName,
 				Protocol:   corev1.ProtocolTCP,

test/config/tls/config-network.yaml

@@ -0,0 +1,26 @@
+# Copyright 2022 The Knative Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: config-network
+  namespace: knative-serving
+  labels:
+    app.kubernetes.io/name: knative-serving
+    app.kubernetes.io/version: devel
+    serving.knative.dev/release: devel
+data:
+  activator-ca: "serving-ca"
+  activator-san: "knative"

test/e2e-common.sh

@@ -50,7 +50,8 @@ export TMP_DIR="${TMP_DIR:-$(mktemp -d -t ci-$(date +%Y-%m-%d-%H-%M-%S)-XXXXXXXX
 readonly E2E_YAML_DIR="${TMP_DIR}/e2e-yaml"
 
 # This the namespace used to install Knative Serving. Use generated UUID as namespace.
-export SYSTEM_NAMESPACE="${SYSTEM_NAMESPACE:-$(uuidgen | tr 'A-Z' 'a-z')}"
+#export SYSTEM_NAMESPACE="${SYSTEM_NAMESPACE:-$(uuidgen | tr 'A-Z' 'a-z')}"
+export SYSTEM_NAMESPACE="knative-serving"
 
 # Keep this in sync with test/ha/ha.go
 readonly REPLICAS=3

test/generate-cert.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+# Copyright 2022 The Knative Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+SERVING_SYSTEM_NAMESPACE=knative-serving
+TEST_NAMESPACE=serving-tests
+out_dir="$(mktemp -d /tmp/certs-XXX)"
+san="knative"
+
+kubectl create ns $SERVING_SYSTEM_NAMESPACE
+
+# Generate Root key and cert.
+openssl req -x509 -sha256 -nodes -days 365 -newkey rsa:2048 -subj '/O=Example/CN=Example' -keyout "${out_dir}"/root.key -out "${out_dir}"/root.crt
+
+# Create server key
+openssl req -out "${out_dir}"/tls.csr -newkey rsa:2048 -nodes -keyout "${out_dir}"/tls.key -subj "/CN=Example/O=Example" -addext "subjectAltName = DNS:$san"
+
+# Create server certs
+openssl x509 -req -extfile <(printf "subjectAltName=DNS:$san") -days 365 -in "${out_dir}"/tls.csr -CA "${out_dir}"/root.crt -CAkey "${out_dir}"/root.key -CAcreateserial -out "${out_dir}"/tls.crt
+
+# Create secret
+kubectl create -n ${SERVING_SYSTEM_NAMESPACE} secret generic serving-ca \
+    --from-file=ca.crt="${out_dir}"/root.crt --dry-run=client -o yaml | kubectl apply -f -
+
+kubectl create -n ${SERVING_SYSTEM_NAMESPACE} secret tls server-certs \
+    --key="${out_dir}"/tls.key \
+    --cert="${out_dir}"/tls.crt --dry-run=client -o yaml | kubectl apply -f -
+
+#kubectl create ns ${TEST_NAMESPACE}
+#kubectl create -n ${TEST_NAMESPACE} secret tls server-certs \
+#    --key="${out_dir}"/tls.key \
+#    --cert="${out_dir}"/tls.crt --dry-run=client -o yaml | kubectl apply -f -

third_party/kourier-latest/kourier.yaml

@@ -297,7 +297,7 @@ spec:
         app: net-kourier-controller
     spec:
       containers:
-        - image: gcr.io/knative-nightly/knative.dev/net-kourier/cmd/kourier@sha256:7677f3bc4bd6d27db485c5abefb596241d2f84d19bce2236712175adfea0d456
+        - image: gcr.io/gcp-compute-engine-223401/kourier-b74c3918b7eee585f87df62ccd297dc8@sha256:5179cf1d0b83bf199f143a59a3c0005ba196b5b2beb8289b56257a992a995fbf
           name: controller
           env:
             - name: CERTS_SECRET_NAMESPACE