diff --git a/.github/workflows/kind-e2e.yaml b/.github/workflows/kind-e2e.yaml index 164d16fe20e5..5f2e48ccf1fd 100644 --- a/.github/workflows/kind-e2e.yaml +++ b/.github/workflows/kind-e2e.yaml @@ -192,6 +192,8 @@ jobs: run: | source ./test/e2e-common.sh + bash ./test/generate-cert.sh + KIND=1 INGRESS_CLASS="${{ matrix.kingress }}.ingress.networking.knative.dev" CLUSTER_DOMAIN="${{ matrix.cluster-suffix }}" @@ -199,6 +201,8 @@ jobs: knative_setup test_setup + kubectl apply -f ./test/config/tls + echo "INGRESS_CLASS=$INGRESS_CLASS" >> $GITHUB_ENV echo "CLUSTER_DOMAIN=$CLUSTER_DOMAIN" >> $GITHUB_ENV echo "SYSTEM_NAMESPACE=$SYSTEM_NAMESPACE" >> $GITHUB_ENV diff --git a/cmd/activator/main.go b/cmd/activator/main.go index 1ae3c9c640d2..63089416954a 100644 --- a/cmd/activator/main.go +++ b/cmd/activator/main.go @@ -68,6 +68,10 @@ const ( // The port on which autoscaler WebSocket server listens. autoscalerPort = ":8080" + + certDirectory = "/var/lib/knative/certs" + certPath = certDirectory + "/tls.crt" + keyPath = certDirectory + "/tls.key" ) type config struct { @@ -241,6 +245,22 @@ func main() { }(name, server) } + // TODO: Use configmap to enable tls mode. + if true { + tlsServers := map[string]*http.Server{ + "https": pkgnet.NewServer(":"+strconv.Itoa(networking.BackendHTTPSPort), ah), + } + + for name, server := range tlsServers { + go func(name string, s *http.Server) { + // Don't forward ErrServerClosed as that indicates we're already shutting down. + if err := s.ListenAndServeTLS(certPath, keyPath); err != nil && !errors.Is(err, http.ErrServerClosed) { + errCh <- fmt.Errorf("%s server failed: %w", name, err) + } + }(name, server) + } + } + // Wait for the signal to drain. select { case <-sigCtx.Done(): diff --git a/config/core/deployments/activator.yaml b/config/core/deployments/activator.yaml index 0a5a5b0dbcd2..989c2b26b83b 100644 --- a/config/core/deployments/activator.yaml +++ b/config/core/deployments/activator.yaml @@ -54,6 +54,11 @@ spec: cpu: 1000m memory: 600Mi + volumeMounts: + - name: server-certs + mountPath: /var/lib/knative/certs + readOnly: true + env: # Run Activator with GC collection when newly generated memory is 500%. - name: GOGC @@ -123,6 +128,11 @@ spec: # connections. terminationGracePeriodSeconds: 600 + volumes: + - name: server-certs + secret: + secretName: server-certs + --- apiVersion: v1 kind: Service @@ -148,6 +158,9 @@ spec: - name: http port: 80 targetPort: 8012 + - name: https + port: 443 + targetPort: 8112 - name: http2 port: 81 targetPort: 8013 diff --git a/pkg/networking/constants.go b/pkg/networking/constants.go index e85118bf939b..0fe1e5ea1048 100644 --- a/pkg/networking/constants.go +++ b/pkg/networking/constants.go @@ -26,6 +26,9 @@ const ( // BackendHTTP2Port is the backend, i.e. `targetPort` that we setup for HTTP/2 services. BackendHTTP2Port = 8013 + // BackendHTTPSPort is the backend. i.e. `targetPort` that we setup for HTTPS services. + BackendHTTPSPort = 8112 + // QueueAdminPort specifies the port number for // health check and lifecycle hooks for queue-proxy. QueueAdminPort = 8022 diff --git a/pkg/reconciler/route/resources/ingress.go b/pkg/reconciler/route/resources/ingress.go index 9700ca41194f..b73010c67289 100644 --- a/pkg/reconciler/route/resources/ingress.go +++ b/pkg/reconciler/route/resources/ingress.go @@ -136,6 +136,11 @@ func makeIngressSpec( rules := make([]netv1alpha1.IngressRule, 0, len(names)) featuresConfig := config.FromContextOrDefaults(ctx).Features + // Use ConfigMap to enable internalTLS. + internalTLS := false + if true { + internalTLS = true + } for _, name := range names { visibilities := []netv1alpha1.IngressVisibility{netv1alpha1.IngressVisibilityClusterLocal} @@ -149,7 +154,7 @@ func makeIngressSpec( return netv1alpha1.IngressSpec{}, err } rule := makeIngressRule(domains, r.Namespace, - visibility, tc.Targets[name], ro.RolloutsByTag(name)) + visibility, tc.Targets[name], ro.RolloutsByTag(name), internalTLS) if featuresConfig.TagHeaderBasedRouting == apicfg.Enabled { if rule.HTTP.Paths[0].AppendHeaders == nil { rule.HTTP.Paths[0].AppendHeaders = make(map[string]string, 1) @@ -171,7 +176,7 @@ func makeIngressSpec( // Since names are sorted `DefaultTarget == ""` is the first one, // so just pass the subslice. rule.HTTP.Paths = append( - makeTagBasedRoutingIngressPaths(r.Namespace, tc, ro, names[1:]), rule.HTTP.Paths...) + makeTagBasedRoutingIngressPaths(r.Namespace, tc, ro, internalTLS, names[1:]), rule.HTTP.Paths...) } else { // If a request is routed by a tag-attached hostname instead of the tag header, // the request may not have the tag header "Knative-Serving-Tag", @@ -290,24 +295,25 @@ func MakeACMEIngressPaths(acmeChallenges []netv1alpha1.HTTP01Challenge, domains func makeIngressRule(domains []string, ns string, visibility netv1alpha1.IngressVisibility, targets traffic.RevisionTargets, - roCfgs []*traffic.ConfigurationRollout) netv1alpha1.IngressRule { + roCfgs []*traffic.ConfigurationRollout, + internalTLS bool) netv1alpha1.IngressRule { return netv1alpha1.IngressRule{ Hosts: domains, Visibility: visibility, HTTP: &netv1alpha1.HTTPIngressRuleValue{ Paths: []netv1alpha1.HTTPIngressPath{ - *makeBaseIngressPath(ns, targets, roCfgs), + *makeBaseIngressPath(ns, targets, roCfgs, internalTLS), }, }, } } // `names` must not include `""` — the DefaultTarget. -func makeTagBasedRoutingIngressPaths(ns string, tc *traffic.Config, ro *traffic.Rollout, names []string) []netv1alpha1.HTTPIngressPath { +func makeTagBasedRoutingIngressPaths(ns string, tc *traffic.Config, ro *traffic.Rollout, internalTLS bool, names []string) []netv1alpha1.HTTPIngressPath { paths := make([]netv1alpha1.HTTPIngressPath, 0, len(names)) for _, name := range names { - path := makeBaseIngressPath(ns, tc.Targets[name], ro.RolloutsByTag(name)) + path := makeBaseIngressPath(ns, tc.Targets[name], ro.RolloutsByTag(name), internalTLS) path.Headers = map[string]netv1alpha1.HeaderMatch{network.TagHeaderName: {Exact: name}} paths = append(paths, *path) } @@ -327,7 +333,7 @@ func rolloutConfig(cfgName string, ros []*traffic.ConfigurationRollout) *traffic } func makeBaseIngressPath(ns string, targets traffic.RevisionTargets, - roCfgs []*traffic.ConfigurationRollout) *netv1alpha1.HTTPIngressPath { + roCfgs []*traffic.ConfigurationRollout, internalTLS bool) *netv1alpha1.HTTPIngressPath { // Optimistically allocate |targets| elements. splits := make([]netv1alpha1.IngressBackendSplit, 0, len(targets)) for _, t := range targets { @@ -339,6 +345,10 @@ func makeBaseIngressPath(ns string, targets traffic.RevisionTargets, if t.LatestRevision != nil && *t.LatestRevision { cfg = rolloutConfig(t.ConfigurationName, roCfgs) } + servicePort := intstr.FromInt(networking.ServicePort(t.Protocol)) + if internalTLS { + servicePort = intstr.FromInt(443) + } if cfg == nil || len(cfg.Revisions) < 2 { // No rollout in progress. splits = append(splits, netv1alpha1.IngressBackendSplit{ @@ -347,7 +357,7 @@ func makeBaseIngressPath(ns string, targets traffic.RevisionTargets, ServiceName: t.RevisionName, // Port on the public service must match port on the activator. // Otherwise, the serverless services can't guarantee seamless positive handoff. - ServicePort: intstr.FromInt(networking.ServicePort(t.Protocol)), + ServicePort: servicePort, }, Percent: int(*t.Percent), AppendHeaders: map[string]string{ @@ -364,7 +374,7 @@ func makeBaseIngressPath(ns string, targets traffic.RevisionTargets, ServiceName: rev.RevisionName, // Port on the public service must match port on the activator. // Otherwise, the serverless services can't guarantee seamless positive handoff. - ServicePort: intstr.FromInt(networking.ServicePort(t.Protocol)), + ServicePort: servicePort, }, Percent: rev.Percent, AppendHeaders: map[string]string{ diff --git a/pkg/reconciler/route/resources/ingress_test.go b/pkg/reconciler/route/resources/ingress_test.go index f955e0b08f56..fd4a65f1676a 100644 --- a/pkg/reconciler/route/resources/ingress_test.go +++ b/pkg/reconciler/route/resources/ingress_test.go @@ -866,8 +866,9 @@ func TestMakeIngressRuleVanilla(t *testing.T) { }, } ro := tc.BuildRollout() + internalTLS := false rule := makeIngressRule(domains, ns, - netv1alpha1.IngressVisibilityExternalIP, targets, ro.RolloutsByTag(traffic.DefaultTarget)) + netv1alpha1.IngressVisibilityExternalIP, targets, ro.RolloutsByTag(traffic.DefaultTarget), internalTLS) expected := netv1alpha1.IngressRule{ Hosts: []string{ "a.com", @@ -919,8 +920,9 @@ func TestMakeIngressRuleZeroPercentTarget(t *testing.T) { }, } ro := tc.BuildRollout() + internalTLS := false rule := makeIngressRule(domains, ns, - netv1alpha1.IngressVisibilityExternalIP, targets, ro.RolloutsByTag(traffic.DefaultTarget)) + netv1alpha1.IngressVisibilityExternalIP, targets, ro.RolloutsByTag(traffic.DefaultTarget), internalTLS) expected := netv1alpha1.IngressRule{ Hosts: []string{"test.org"}, HTTP: &netv1alpha1.HTTPIngressRuleValue{ @@ -968,9 +970,10 @@ func TestMakeIngressRuleTwoTargets(t *testing.T) { }, } ro := tc.BuildRollout() + internalTLS := false domains := []string{"test.org"} rule := makeIngressRule(domains, ns, netv1alpha1.IngressVisibilityExternalIP, - targets, ro.RolloutsByTag("a-tag")) + targets, ro.RolloutsByTag("a-tag"), internalTLS) expected := netv1alpha1.IngressRule{ Hosts: []string{"test.org"}, HTTP: &netv1alpha1.HTTPIngressRuleValue{ diff --git a/pkg/reconciler/serverlessservice/resources/services.go b/pkg/reconciler/serverlessservice/resources/services.go index a6f51f032986..1c07531b72c8 100644 --- a/pkg/reconciler/serverlessservice/resources/services.go +++ b/pkg/reconciler/serverlessservice/resources/services.go @@ -53,16 +53,32 @@ func MakePublicService(sks *v1alpha1.ServerlessService) *corev1.Service { OwnerReferences: []metav1.OwnerReference{*kmeta.NewControllerRef(sks)}, }, Spec: corev1.ServiceSpec{ - Ports: []corev1.ServicePort{{ - Name: pkgnet.ServicePortName(sks.Spec.ProtocolType), - Protocol: corev1.ProtocolTCP, - Port: int32(pkgnet.ServicePort(sks.Spec.ProtocolType)), - TargetPort: targetPort(sks), - }}, + Ports: makePublicServicePorts(sks), }, } } +func makePublicServicePorts(sks *v1alpha1.ServerlessService) []corev1.ServicePort { + ports := []corev1.ServicePort{{ + Name: pkgnet.ServicePortName(sks.Spec.ProtocolType), + Protocol: corev1.ProtocolTCP, + Port: int32(pkgnet.ServicePort(sks.Spec.ProtocolType)), + TargetPort: targetPort(sks), + }} + + // TODO: Use annotation or sks.spec whether add HTTPS port or not. + if true { + p := corev1.ServicePort{ + Name: pkgnet.ServicePortNameHTTPS, + Protocol: corev1.ProtocolTCP, + Port: pkgnet.ServiceHTTPSPort, + TargetPort: intstr.FromInt(networking.BackendHTTPSPort), + } + ports = append(ports, p) + } + return ports +} + // MakePublicEndpoints constructs a K8s Endpoints that is not backed a selector // and will be manually reconciled by the SKS controller. func MakePublicEndpoints(sks *v1alpha1.ServerlessService, src *corev1.Endpoints) *corev1.Endpoints { @@ -99,11 +115,20 @@ func filterSubsetPorts(targetPort int32, subsets []corev1.EndpointSubset) []core ret := make([]corev1.EndpointSubset, len(subsets)) for i, sss := range subsets { sst := sss.DeepCopy() + ssts := sss.DeepCopy() // Find the port we care about and remove all others. for j, p := range sst.Ports { if p.Port == targetPort { sst.Ports = sst.Ports[j : j+1] - break + } + } + + // TODO: Use annotation or configmap to add HTTPS port. + if true { + for j, p := range ssts.Ports { + if p.Port == networking.BackendHTTPSPort { + sst.Ports = append(sst.Ports, ssts.Ports[j:j+1]...) + } } } ret[i] = *sst @@ -134,6 +159,12 @@ func MakePrivateService(sks *v1alpha1.ServerlessService, selector map[string]str // This one is matching the public one, since this is the // port queue-proxy listens on. TargetPort: targetPort(sks), + }, { + // TODO: Add https port only when tls mode is enabled? + Name: pkgnet.ServicePortNameHTTPS, + Protocol: corev1.ProtocolTCP, + Port: pkgnet.ServiceHTTPSPort, + TargetPort: intstr.FromInt(networking.BackendHTTPSPort), }, { Name: servingv1.AutoscalingQueueMetricsPortName, Protocol: corev1.ProtocolTCP, diff --git a/test/config/tls/config-network.yaml b/test/config/tls/config-network.yaml new file mode 100644 index 000000000000..51c849bfcad3 --- /dev/null +++ b/test/config/tls/config-network.yaml @@ -0,0 +1,26 @@ +# Copyright 2022 The Knative Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: config-network + namespace: knative-serving + labels: + app.kubernetes.io/name: knative-serving + app.kubernetes.io/version: devel + serving.knative.dev/release: devel +data: + activator-ca: "serving-ca" + activator-san: "knative" diff --git a/test/e2e-common.sh b/test/e2e-common.sh index 9546d0131424..7dd54bc563fb 100644 --- a/test/e2e-common.sh +++ b/test/e2e-common.sh @@ -50,7 +50,8 @@ export TMP_DIR="${TMP_DIR:-$(mktemp -d -t ci-$(date +%Y-%m-%d-%H-%M-%S)-XXXXXXXX readonly E2E_YAML_DIR="${TMP_DIR}/e2e-yaml" # This the namespace used to install Knative Serving. Use generated UUID as namespace. -export SYSTEM_NAMESPACE="${SYSTEM_NAMESPACE:-$(uuidgen | tr 'A-Z' 'a-z')}" +#export SYSTEM_NAMESPACE="${SYSTEM_NAMESPACE:-$(uuidgen | tr 'A-Z' 'a-z')}" +export SYSTEM_NAMESPACE="knative-serving" # Keep this in sync with test/ha/ha.go readonly REPLICAS=3 diff --git a/test/generate-cert.sh b/test/generate-cert.sh new file mode 100755 index 000000000000..b08bc422b72c --- /dev/null +++ b/test/generate-cert.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +# Copyright 2022 The Knative Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SERVING_SYSTEM_NAMESPACE=knative-serving +TEST_NAMESPACE=serving-tests +out_dir="$(mktemp -d /tmp/certs-XXX)" +san="knative" + +kubectl create ns $SERVING_SYSTEM_NAMESPACE + +# Generate Root key and cert. +openssl req -x509 -sha256 -nodes -days 365 -newkey rsa:2048 -subj '/O=Example/CN=Example' -keyout "${out_dir}"/root.key -out "${out_dir}"/root.crt + +# Create server key +openssl req -out "${out_dir}"/tls.csr -newkey rsa:2048 -nodes -keyout "${out_dir}"/tls.key -subj "/CN=Example/O=Example" -addext "subjectAltName = DNS:$san" + +# Create server certs +openssl x509 -req -extfile <(printf "subjectAltName=DNS:$san") -days 365 -in "${out_dir}"/tls.csr -CA "${out_dir}"/root.crt -CAkey "${out_dir}"/root.key -CAcreateserial -out "${out_dir}"/tls.crt + +# Create secret +kubectl create -n ${SERVING_SYSTEM_NAMESPACE} secret generic serving-ca \ + --from-file=ca.crt="${out_dir}"/root.crt --dry-run=client -o yaml | kubectl apply -f - + +kubectl create -n ${SERVING_SYSTEM_NAMESPACE} secret tls server-certs \ + --key="${out_dir}"/tls.key \ + --cert="${out_dir}"/tls.crt --dry-run=client -o yaml | kubectl apply -f - + +#kubectl create ns ${TEST_NAMESPACE} +#kubectl create -n ${TEST_NAMESPACE} secret tls server-certs \ +# --key="${out_dir}"/tls.key \ +# --cert="${out_dir}"/tls.crt --dry-run=client -o yaml | kubectl apply -f -