diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9d6d7f159..6d1e75fa4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -56,7 +56,7 @@ jobs: - name: golangci-lint uses: golangci/golangci-lint-action@v6 with: - version: v1.55 + version: v1.59 skip-pkg-cache: true codegen: diff --git a/.golangci.yml b/.golangci.yml index cf493ba89..daf3d1adf 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -3,6 +3,8 @@ run: timeout: 20m linters: + disable: + - errcheck enable: - goconst - gocyclo @@ -22,12 +24,18 @@ linters-settings: rules: - name: unused-parameter disabled: true + gosec: + exclude-generated: true + severity: medium + confidence: medium issues: exclude-dirs: - pkg/gen + - pkg/bugreport exclude-files: - "zz_generated.deepcopy.go$" + - "zz_generated.register.go$" exclude-rules: # Ignore error for ginkgo and gomega dot imports - linters: diff --git a/charts/gateway/README.md b/charts/gateway/README.md index 59e9bb1d5..503a8ede2 100644 --- a/charts/gateway/README.md +++ b/charts/gateway/README.md @@ -59,44 +59,50 @@ The following table lists the configurable parameters of the fsm chart and their | Key | Type | Default | Description | |-----|------|---------|-------------| -| fsm.fsmGateway.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].key | string | `"kubernetes.io/os"` | | -| fsm.fsmGateway.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].operator | string | `"In"` | | -| fsm.fsmGateway.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].values[0] | string | `"linux"` | | -| fsm.fsmGateway.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[1].key | string | `"kubernetes.io/arch"` | | -| fsm.fsmGateway.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[1].operator | string | `"In"` | | -| fsm.fsmGateway.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[1].values[0] | string | `"amd64"` | | -| fsm.fsmGateway.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[1].values[1] | string | `"arm64"` | | -| fsm.fsmGateway.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.labelSelector.matchExpressions[0].key | string | `"app"` | | -| fsm.fsmGateway.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.labelSelector.matchExpressions[0].operator | string | `"In"` | | -| fsm.fsmGateway.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.labelSelector.matchExpressions[0].values[0] | string | `"fsm-gateway"` | | -| fsm.fsmGateway.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.topologyKey | string | `"kubernetes.io/hostname"` | | -| fsm.fsmGateway.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].weight | int | `100` | | -| fsm.fsmGateway.autoScale | object | `{"behavior":{"scaleDown":{"policies":[{"periodSeconds":60,"type":"Pods","value":1},{"periodSeconds":60,"type":"Percent","value":10}],"selectPolicy":"Min","stabilizationWindowSeconds":300},"scaleUp":{"policies":[{"periodSeconds":15,"type":"Percent","value":100},{"periodSeconds":15,"type":"Pods","value":2}],"selectPolicy":"Max","stabilizationWindowSeconds":0}},"cpu":{"targetAverageUtilization":80},"enable":false,"maxReplicas":10,"memory":{"targetAverageUtilization":80},"metrics":[{"resource":{"name":"cpu","target":{"averageUtilization":80,"type":"Utilization"}},"type":"Resource"},{"resource":{"name":"memory","target":{"averageUtilization":80,"type":"Utilization"}},"type":"Resource"}],"minReplicas":1}` | Auto scale configuration | -| fsm.fsmGateway.autoScale.behavior | object | `{"scaleDown":{"policies":[{"periodSeconds":60,"type":"Pods","value":1},{"periodSeconds":60,"type":"Percent","value":10}],"selectPolicy":"Min","stabilizationWindowSeconds":300},"scaleUp":{"policies":[{"periodSeconds":15,"type":"Percent","value":100},{"periodSeconds":15,"type":"Pods","value":2}],"selectPolicy":"Max","stabilizationWindowSeconds":0}}` | Auto scale behavior, for v2 API | -| fsm.fsmGateway.autoScale.cpu | object | `{"targetAverageUtilization":80}` | Auto scale cpu metrics, for v2beta2 API | -| fsm.fsmGateway.autoScale.cpu.targetAverageUtilization | int | `80` | Average target CPU utilization (%) | -| fsm.fsmGateway.autoScale.enable | bool | `false` | Enable Autoscale | -| fsm.fsmGateway.autoScale.maxReplicas | int | `10` | Maximum replicas for autoscale | -| fsm.fsmGateway.autoScale.memory | object | `{"targetAverageUtilization":80}` | Auto scale memory metrics, for v2beta2 API | -| fsm.fsmGateway.autoScale.memory.targetAverageUtilization | int | `80` | Average target memory utilization (%) | -| fsm.fsmGateway.autoScale.metrics | list | `[{"resource":{"name":"cpu","target":{"averageUtilization":80,"type":"Utilization"}},"type":"Resource"},{"resource":{"name":"memory","target":{"averageUtilization":80,"type":"Utilization"}},"type":"Resource"}]` | Auto scale metrics, for v2 API | -| fsm.fsmGateway.autoScale.minReplicas | int | `1` | Minimum replicas for autoscale | -| fsm.fsmGateway.env[0].name | string | `"GIN_MODE"` | | -| fsm.fsmGateway.env[0].value | string | `"release"` | | -| fsm.fsmGateway.initResources | object | `{"limits":{"cpu":"500m","memory":"512M"},"requests":{"cpu":"200m","memory":"128M"}}` | initContainer resource configuration | -| fsm.fsmGateway.logLevel | string | `"info"` | | -| fsm.fsmGateway.nodeSelector | object | `{}` | Node selector applied to control plane pods. | -| fsm.fsmGateway.podAnnotations | object | `{}` | FSM Gateway Controller's pod annotations | -| fsm.fsmGateway.podDisruptionBudget | object | `{"enabled":false,"minAvailable":1}` | Pod disruption budget configuration | -| fsm.fsmGateway.podDisruptionBudget.enabled | bool | `false` | Enable Pod Disruption Budget | -| fsm.fsmGateway.podDisruptionBudget.minAvailable | int | `1` | Minimum number of pods that must be available | -| fsm.fsmGateway.podLabels | object | `{}` | FSM Gateway Controller's pod labels | -| fsm.fsmGateway.podSecurityContext | object | `{"runAsGroup":65532,"runAsNonRoot":true,"runAsUser":65532,"seccompProfile":{"type":"RuntimeDefault"}}` | FSM Gateway Controller's pod security context | -| fsm.fsmGateway.replicas | int | `1` | | -| fsm.fsmGateway.resources | object | `{"limits":{"cpu":"2","memory":"1G"},"requests":{"cpu":"0.5","memory":"128M"}}` | FSM Gateway's container resource parameters. | -| fsm.fsmGateway.securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]}}` | FSM Gateway Controller's container security context | -| fsm.fsmGateway.tolerations | list | `[]` | Node tolerations applied to control plane pods. The specified tolerations allow pods to schedule onto nodes with matching taints. | | fsm.fsmNamespace | string | `""` | Namespace to deploy FSM in. If not specified, the Helm release namespace is used. | +| fsm.gateway.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].key | string | `"kubernetes.io/os"` | | +| fsm.gateway.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].operator | string | `"In"` | | +| fsm.gateway.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].values[0] | string | `"linux"` | | +| fsm.gateway.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[1].key | string | `"kubernetes.io/arch"` | | +| fsm.gateway.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[1].operator | string | `"In"` | | +| fsm.gateway.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[1].values[0] | string | `"amd64"` | | +| fsm.gateway.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[1].values[1] | string | `"arm64"` | | +| fsm.gateway.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.labelSelector.matchExpressions[0].key | string | `"app"` | | +| fsm.gateway.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.labelSelector.matchExpressions[0].operator | string | `"In"` | | +| fsm.gateway.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.labelSelector.matchExpressions[0].values[0] | string | `"fsm-gateway"` | | +| fsm.gateway.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.topologyKey | string | `"kubernetes.io/hostname"` | | +| fsm.gateway.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].weight | int | `100` | | +| fsm.gateway.autoScale | object | `{"behavior":{"scaleDown":{"policies":[{"periodSeconds":60,"type":"Pods","value":1},{"periodSeconds":60,"type":"Percent","value":10}],"selectPolicy":"Min","stabilizationWindowSeconds":300},"scaleUp":{"policies":[{"periodSeconds":15,"type":"Percent","value":100},{"periodSeconds":15,"type":"Pods","value":2}],"selectPolicy":"Max","stabilizationWindowSeconds":0}},"cpu":{"targetAverageUtilization":80},"enable":false,"maxReplicas":10,"memory":{"targetAverageUtilization":80},"metrics":[{"resource":{"name":"cpu","target":{"averageUtilization":80,"type":"Utilization"}},"type":"Resource"},{"resource":{"name":"memory","target":{"averageUtilization":80,"type":"Utilization"}},"type":"Resource"}],"minReplicas":1}` | Auto scale configuration | +| fsm.gateway.autoScale.behavior | object | `{"scaleDown":{"policies":[{"periodSeconds":60,"type":"Pods","value":1},{"periodSeconds":60,"type":"Percent","value":10}],"selectPolicy":"Min","stabilizationWindowSeconds":300},"scaleUp":{"policies":[{"periodSeconds":15,"type":"Percent","value":100},{"periodSeconds":15,"type":"Pods","value":2}],"selectPolicy":"Max","stabilizationWindowSeconds":0}}` | Auto scale behavior, for v2 API | +| fsm.gateway.autoScale.cpu | object | `{"targetAverageUtilization":80}` | Auto scale cpu metrics, for v2beta2 API | +| fsm.gateway.autoScale.cpu.targetAverageUtilization | int | `80` | Average target CPU utilization (%) | +| fsm.gateway.autoScale.enable | bool | `false` | Enable Autoscale | +| fsm.gateway.autoScale.maxReplicas | int | `10` | Maximum replicas for autoscale | +| fsm.gateway.autoScale.memory | object | `{"targetAverageUtilization":80}` | Auto scale memory metrics, for v2beta2 API | +| fsm.gateway.autoScale.memory.targetAverageUtilization | int | `80` | Average target memory utilization (%) | +| fsm.gateway.autoScale.metrics | list | `[{"resource":{"name":"cpu","target":{"averageUtilization":80,"type":"Utilization"}},"type":"Resource"},{"resource":{"name":"memory","target":{"averageUtilization":80,"type":"Utilization"}},"type":"Resource"}]` | Auto scale metrics, for v2 API | +| fsm.gateway.autoScale.minReplicas | int | `1` | Minimum replicas for autoscale | +| fsm.gateway.env[0].name | string | `"GIN_MODE"` | | +| fsm.gateway.env[0].value | string | `"release"` | | +| fsm.gateway.infrastructure | object | `{"annotations":{},"labels":{}}` | Gateway's infrastructure, override by gateway.spec.infrastructure.annotations and gateway.spec.infrastructure.labels -- NOT override by parameterRef | +| fsm.gateway.initResources | object | `{"limits":{"cpu":"500m","memory":"512M"},"requests":{"cpu":"200m","memory":"128M"}}` | initContainer resource configuration | +| fsm.gateway.listeners | object | `{}` | Gateway's listeners, not overridable by parameterRef | +| fsm.gateway.logLevel | string | `"info"` | | +| fsm.gateway.name | string | `"UNKNOWN"` | Gateway's name, not overridable by parameterRef | +| fsm.gateway.namespace | string | `"default"` | Gateway's namespace, not overridable by parameterRef | +| fsm.gateway.nodePorts | list | `[]` | NodePort service configuration nodePorts: - port: 80 nodePort: 30080 - port: 443 nodePort: 30443 - port: 53 nodePort: 30053 | +| fsm.gateway.nodeSelector | object | `{}` | Node selector applied to control plane pods. | +| fsm.gateway.podAnnotations | object | `{}` | FSM Gateway Controller's pod annotations | +| fsm.gateway.podDisruptionBudget | object | `{"enabled":false,"minAvailable":1}` | Pod disruption budget configuration | +| fsm.gateway.podDisruptionBudget.enabled | bool | `false` | Enable Pod Disruption Budget | +| fsm.gateway.podDisruptionBudget.minAvailable | int | `1` | Minimum number of pods that must be available | +| fsm.gateway.podLabels | object | `{}` | FSM Gateway Controller's pod labels | +| fsm.gateway.podSecurityContext | object | `{"runAsGroup":65532,"runAsNonRoot":true,"runAsUser":65532,"seccompProfile":{"type":"RuntimeDefault"}}` | FSM Gateway Controller's pod security context | +| fsm.gateway.replicas | int | `1` | FSM Gateway's replica count | +| fsm.gateway.resources | object | `{"limits":{"cpu":"2","memory":"1G"},"requests":{"cpu":"0.5","memory":"128M"}}` | FSM Gateway's container resource parameters. | +| fsm.gateway.securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]}}` | FSM Gateway Controller's container security context | +| fsm.gateway.serviceType | string | `"LoadBalancer"` | FSM Gateway's service type, only LoadBalancer and NodePort are supported | +| fsm.gateway.tolerations | list | `[]` | Node tolerations applied to control plane pods. The specified tolerations allow pods to schedule onto nodes with matching taints. | | fsm.image.digest | object | `{"fsmCurl":"","fsmGateway":""}` | Image digest (defaults to latest compatible tag) | | fsm.image.digest.fsmCurl | string | `""` | fsm-curl's image digest | | fsm.image.digest.fsmGateway | string | `""` | fsm-gateway's image digest | diff --git a/charts/gateway/templates/_helpers.tpl b/charts/gateway/templates/_helpers.tpl index 74523359f..530b75a54 100644 --- a/charts/gateway/templates/_helpers.tpl +++ b/charts/gateway/templates/_helpers.tpl @@ -2,7 +2,7 @@ ServiceAccountName - GatewayAPI */}} {{- define "fsm.gateway.serviceAccountName" -}} -{{ printf "fsm-gateway-%s" .Values.gwy.metadata.namespace }} +{{ printf "fsm-gateway-%s-%s" .Values.fsm.gateway.namespace .Values.fsm.gateway.name }} {{- end }} {{/* fsm-gateway image */}} diff --git a/charts/gateway/templates/deployment.yaml b/charts/gateway/templates/deployment.yaml index 6f2e015a3..1524c3699 100644 --- a/charts/gateway/templates/deployment.yaml +++ b/charts/gateway/templates/deployment.yaml @@ -2,29 +2,30 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ printf "fsm-gateway-%s" .Values.gwy.metadata.namespace }} - namespace: {{ .Values.gwy.metadata.namespace }} - {{- if .Values.gwy.spec.infrastructure }} - {{- with .Values.gwy.spec.infrastructure.annotations }} + name: {{ printf "fsm-gateway-%s-%s" .Values.fsm.gateway.namespace .Values.fsm.gateway.name }} + namespace: {{ .Values.fsm.gateway.namespace }} + {{- if .Values.fsm.gateway.infrastructure }} + {{- with .Values.fsm.gateway.infrastructure.annotations }} annotations: {{- toYaml . | nindent 4 }} {{- end }} {{- end }} labels: {{- include "fsm.labels" . | nindent 4 }} - {{- if .Values.gwy.spec.infrastructure }} - {{- with .Values.gwy.spec.infrastructure.labels }} + {{- if .Values.fsm.gateway.infrastructure }} + {{- with .Values.fsm.gateway.infrastructure.labels }} {{- toYaml . | nindent 4 }} {{- end }} {{- end }} app: fsm-gateway meshName: {{ .Values.fsm.meshName }} spec: - replicas: {{ default 1 .Values.fsm.fsmGateway.replicas }} + replicas: {{ default 1 .Values.fsm.gateway.replicas }} selector: matchLabels: app: fsm-gateway - gateway.flomesh.io/ns: {{ .Values.gwy.metadata.namespace }} + gateway.flomesh.io/ns: {{ .Values.fsm.gateway.namespace }} + gateway.flomesh.io/name: {{ .Values.fsm.gateway.name }} strategy: rollingUpdate: maxSurge: 1 @@ -34,23 +35,24 @@ spec: metadata: labels: app: fsm-gateway - gateway.flomesh.io/ns: {{ .Values.gwy.metadata.namespace }} - {{- with .Values.fsm.fsmGateway.podLabels }} + gateway.flomesh.io/ns: {{ .Values.fsm.gateway.namespace }} + gateway.flomesh.io/name: {{ .Values.fsm.gateway.name }} + {{- with .Values.fsm.gateway.podLabels }} {{- toYaml . | nindent 8 }} {{- end }} - {{- if .Values.gwy.spec.infrastructure }} - {{- with .Values.gwy.spec.infrastructure.labels }} + {{- if .Values.fsm.gateway.infrastructure }} + {{- with .Values.fsm.gateway.infrastructure.labels }} {{- toYaml . | nindent 8 }} {{- end }} {{- end }} annotations: prometheus.io/scrape: 'true' prometheus.io/port: '9091' - {{- with .Values.fsm.fsmGateway.podAnnotations }} + {{- with .Values.fsm.gateway.podAnnotations }} {{- toYaml . | nindent 8 }} {{- end }} - {{- if .Values.gwy.spec.infrastructure }} - {{- with .Values.gwy.spec.infrastructure.annotations }} + {{- if .Values.fsm.gateway.infrastructure }} + {{- with .Values.fsm.gateway.infrastructure.annotations }} {{- toYaml . | nindent 8 }} {{- end }} {{- end }} @@ -71,8 +73,8 @@ spec: - --retry-delay - "5" resources: - {{- toYaml .Values.fsm.fsmGateway.initResources | nindent 10 }} - {{- with .Values.fsm.fsmGateway.securityContext }} + {{- toYaml .Values.fsm.gateway.initResources | nindent 10 }} + {{- with .Values.fsm.gateway.securityContext }} securityContext: {{- toYaml . | nindent 10 }} {{- end }} @@ -81,19 +83,21 @@ spec: image: {{ include "fsmGateway.image" . }} imagePullPolicy: {{ .Values.fsm.image.pullPolicy }} ports: - {{- range $listener := .Values.listeners }} + {{- range $listener := .Values.fsm.gateway.listeners }} - name: {{ $listener.name }} containerPort: {{ ternary (add 60000 $listener.port) $listener.port (lt (int $listener.port) 1024)}} {{- end }} - name: health containerPort: 8081 args: - - --verbosity={{ .Values.fsm.fsmGateway.logLevel }} + - --verbosity={{ .Values.fsm.gateway.logLevel }} - --fsm-namespace={{ .Values.fsm.fsmNamespace }} - --fsm-version={{ .Chart.AppVersion }} - --mesh-name={{ .Values.fsm.meshName }} + - --gateway-namespace={{ .Values.fsm.gateway.namespace }} + - --gateway-name={{ .Values.fsm.gateway.name }} resources: - {{- toYaml .Values.fsm.fsmGateway.resources | nindent 10 }} + {{- toYaml .Values.fsm.gateway.resources | nindent 10 }} env: - name: FSM_NAMESPACE value: {{ .Values.fsm.fsmNamespace }} @@ -105,10 +109,10 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - {{- with .Values.fsm.fsmGateway.env }} + {{- with .Values.fsm.gateway.env }} {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.fsm.fsmGateway.securityContext }} + {{- with .Values.fsm.gateway.securityContext }} securityContext: {{- toYaml . | nindent 10 }} {{- end }} @@ -123,7 +127,7 @@ spec: tcpSocket: port: 9091 terminationGracePeriodSeconds: 60 - {{- with .Values.fsm.fsmGateway.podSecurityContext }} + {{- with .Values.fsm.gateway.podSecurityContext }} securityContext: {{- toYaml . | nindent 8 }} {{- end }} @@ -132,7 +136,7 @@ spec: imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.fsm.fsmGateway.nodeSelector }} + {{- with .Values.fsm.gateway.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} {{- end }} @@ -162,10 +166,14 @@ spec: - key: gateway.flomesh.io/ns operator: In values: - - {{ .Values.gwy.metadata.namespace }} + - {{ .Values.fsm.gateway.namespace }} + - key: gateway.flomesh.io/name + operator: In + values: + - {{ .Values.fsm.gateway.name }} topologyKey: kubernetes.io/hostname weight: 100 - {{- with .Values.fsm.fsmGateway.tolerations }} + {{- with .Values.fsm.gateway.tolerations }} tolerations: {{- toYaml . | nindent 8 }} {{- end }} diff --git a/charts/gateway/templates/hpa.yaml b/charts/gateway/templates/hpa.yaml index 711fa8401..ff3dc2aea 100644 --- a/charts/gateway/templates/hpa.yaml +++ b/charts/gateway/templates/hpa.yaml @@ -1,22 +1,22 @@ -{{- if .Values.fsm.fsmGateway.autoScale.enable }} +{{- if .Values.fsm.gateway.autoScale.enable }} {{- if (semverCompare ">=1.23-0" .Capabilities.KubeVersion.GitVersion) }} apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: - name: {{ printf "fsm-gateway-%s-hpa" .Values.gwy.metadata.namespace }} - namespace: {{ .Values.gwy.metadata.namespace }} + name: {{ printf "fsm-gateway-%s-%s-hpa" .Values.fsm.gateway.namespace .Values.fsm.gateway.name }} + namespace: {{ .Values.fsm.gateway.namespace }} spec: scaleTargetRef: apiVersion: apps/v1 kind: Deployment - name: {{ printf "fsm-gateway-%s" .Values.gwy.metadata.namespace }} - minReplicas: {{.Values.fsm.fsmGateway.autoScale.minReplicas}} - maxReplicas: {{.Values.fsm.fsmGateway.autoScale.maxReplicas}} - {{- with .Values.fsm.fsmGateway.autoScale.metrics }} + name: {{ printf "fsm-gateway-%s-%s" .Values.fsm.gateway.namespace .Values.fsm.gateway.name }} + minReplicas: {{.Values.fsm.gateway.autoScale.minReplicas}} + maxReplicas: {{.Values.fsm.gateway.autoScale.maxReplicas}} + {{- with .Values.fsm.gateway.autoScale.metrics }} metrics: {{- toYaml . | nindent 4 }} {{- end }} - {{- with .Values.fsm.fsmGateway.autoScale.behavior }} + {{- with .Values.fsm.gateway.autoScale.behavior }} behavior: {{- toYaml . | nindent 4 }} {{- end }} @@ -24,27 +24,27 @@ spec: apiVersion: autoscaling/v2beta2 kind: HorizontalPodAutoscaler metadata: - name: {{ printf "fsm-gateway-%s-hpa" .Values.gwy.metadata.namespace }} - namespace: {{ .Values.gwy.metadata.namespace }} + name: {{ printf "fsm-gateway-%s-%s-hpa" .Values.fsm.gateway.namespace .Values.fsm.gateway.name }} + namespace: {{ .Values.fsm.gateway.namespace }} spec: scaleTargetRef: apiVersion: apps/v1 kind: Deployment - name: {{ printf "fsm-gateway-%s" .Values.gwy.metadata.namespace }} - minReplicas: {{.Values.fsm.fsmGateway.autoScale.minReplicas}} - maxReplicas: {{.Values.fsm.fsmGateway.autoScale.maxReplicas}} + name: {{ printf "fsm-gateway-%s-%s" .Values.fsm.gateway.namespace .Values.fsm.gateway.name }} + minReplicas: {{.Values.fsm.gateway.autoScale.minReplicas}} + maxReplicas: {{.Values.fsm.gateway.autoScale.maxReplicas}} metrics: - type: Resource resource: name: cpu target: type: Utilization - averageUtilization: {{.Values.fsm.fsmGateway.autoScale.cpu.targetAverageUtilization}} + averageUtilization: {{.Values.fsm.gateway.autoScale.cpu.targetAverageUtilization}} - type: Resource resource: name: memory target: type: Utilization - averageUtilization: {{.Values.fsm.fsmGateway.autoScale.memory.targetAverageUtilization}} + averageUtilization: {{.Values.fsm.gateway.autoScale.memory.targetAverageUtilization}} {{- end }} {{- end }} \ No newline at end of file diff --git a/charts/gateway/templates/pod-disruption-budget.yaml b/charts/gateway/templates/pod-disruption-budget.yaml index 47f35da88..e3d9a25f7 100644 --- a/charts/gateway/templates/pod-disruption-budget.yaml +++ b/charts/gateway/templates/pod-disruption-budget.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.fsm.fsmGateway.podDisruptionBudget.enabled}} +{{- if and .Values.fsm.gateway.podDisruptionBudget.enabled}} {{- if (semverCompare ">=1.21-0" .Capabilities.KubeVersion.GitVersion) }} apiVersion: policy/v1 {{- else }} @@ -6,15 +6,17 @@ apiVersion: policy/v1beta1 {{- end }} kind: PodDisruptionBudget metadata: - name: {{ printf "fsm-gateway-%s-pdb" .Values.gwy.metadata.namespace }} - namespace: {{ .Values.gwy.metadata.namespace }} + name: {{ printf "fsm-gateway-%s-%s-pdb" .Values.fsm.gateway.namespace .Values.fsm.gateway.name }} + namespace: {{ .Values.fsm.gateway.namespace }} labels: app: fsm-gateway - gateway.flomesh.io/ns: {{ .Values.gwy.metadata.namespace }} + gateway.flomesh.io/ns: {{ .Values.fsm.gateway.namespace }} + gateway.flomesh.io/name: {{ .Values.fsm.gateway.name }} spec: - minAvailable: {{ .Values.fsm.fsmGateway.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.fsm.gateway.podDisruptionBudget.minAvailable }} selector: matchLabels: app: fsm-gateway - gateway.flomesh.io/ns: {{ .Values.gwy.metadata.namespace }} + gateway.flomesh.io/ns: {{ .Values.fsm.gateway.namespace }} + gateway.flomesh.io/name: {{ .Values.fsm.gateway.name }} {{- end }} \ No newline at end of file diff --git a/charts/gateway/templates/role.yaml b/charts/gateway/templates/role.yaml index 2e20f9add..46cb435f9 100644 --- a/charts/gateway/templates/role.yaml +++ b/charts/gateway/templates/role.yaml @@ -3,7 +3,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: fsm-gateway-role - namespace: {{ .Values.gwy.metadata.namespace }} + namespace: {{ .Values.fsm.gateway.namespace }} labels: {{- include "fsm.labels" . | nindent 4 }} app: fsm-gateway diff --git a/charts/gateway/templates/rolebinding.yaml b/charts/gateway/templates/rolebinding.yaml index 794c3ba3d..e6d934d1c 100644 --- a/charts/gateway/templates/rolebinding.yaml +++ b/charts/gateway/templates/rolebinding.yaml @@ -3,7 +3,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: fsm-gateway-rolebinding - namespace: {{ .Values.gwy.metadata.namespace }} + namespace: {{ .Values.fsm.gateway.namespace }} labels: {{- include "fsm.labels" . | nindent 4 }} app: fsm-gateway @@ -14,12 +14,12 @@ roleRef: subjects: - kind: ServiceAccount name: {{ include "fsm.gateway.serviceAccountName" . }} - namespace: {{ .Values.gwy.metadata.namespace }} + namespace: {{ .Values.fsm.gateway.namespace }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: {{ printf "fsm-mesh-config-rolebinding-%s" .Values.gwy.metadata.namespace }} + name: {{ printf "fsm-mesh-config-rolebinding-%s" .Values.fsm.gateway.namespace }} namespace: {{ .Values.fsm.fsmNamespace }} labels: {{- include "fsm.labels" . | nindent 4 }} @@ -31,5 +31,5 @@ roleRef: subjects: - kind: ServiceAccount name: {{ include "fsm.gateway.serviceAccountName" . }} - namespace: {{ .Values.gwy.metadata.namespace }} + namespace: {{ .Values.fsm.gateway.namespace }} {{- end }} \ No newline at end of file diff --git a/charts/gateway/templates/service-tcp.yaml b/charts/gateway/templates/service-tcp.yaml index 76b9cfada..36372866f 100644 --- a/charts/gateway/templates/service-tcp.yaml +++ b/charts/gateway/templates/service-tcp.yaml @@ -2,35 +2,46 @@ apiVersion: v1 kind: Service metadata: - name: {{ printf "fsm-gateway-%s-tcp" .Values.gwy.metadata.namespace }} - namespace: {{ .Values.gwy.metadata.namespace }} - {{- if .Values.gwy.spec.infrastructure }} - {{- with .Values.gwy.spec.infrastructure.annotations }} + name: {{ printf "fsm-gateway-%s-%s-tcp" .Values.fsm.gateway.namespace .Values.fsm.gateway.name }} + namespace: {{ .Values.fsm.gateway.namespace }} + {{- if .Values.fsm.gateway.infrastructure }} + {{- with .Values.fsm.gateway.infrastructure.annotations }} annotations: {{- toYaml . | nindent 4 }} {{- end }} {{- end }} labels: {{- include "fsm.labels" . | nindent 4 }} - {{- if .Values.gwy.spec.infrastructure }} - {{- with .Values.gwy.spec.infrastructure.labels }} + {{- if .Values.fsm.gateway.infrastructure }} + {{- with .Values.fsm.gateway.infrastructure.labels }} {{- toYaml . | nindent 4 }} {{- end }} {{- end }} app: fsm-gateway - gateway.flomesh.io/ns: {{ .Values.gwy.metadata.namespace }} + gateway.flomesh.io/ns: {{ .Values.fsm.gateway.namespace }} + gateway.flomesh.io/name: {{ .Values.fsm.gateway.name }} spec: - type: LoadBalancer + type: {{ .Values.fsm.gateway.serviceType }} + {{- $setNodePorts := (and (eq .Values.fsm.gateway.serviceType "NodePort") .Values.fsm.gateway.nodePorts)}} ports: - {{- range $listener := .Values.listeners }} - {{- if ne $listener.protocol "UDP" }} - - name: {{ $listener.name }} - port: {{ $listener.port }} - targetPort: {{ ternary (add 60000 $listener.port) $listener.port (lt (int $listener.port) 1024)}} + {{- range .Values.fsm.gateway.listeners }} + {{- if ne .protocol "UDP" }} + {{- $listenerPort := .port }} + - name: {{ .name }} + port: {{ $listenerPort }} + targetPort: {{ ternary (add 60000 $listenerPort) $listenerPort (lt (int $listenerPort) 1024)}} protocol: TCP + {{- if $setNodePorts }} + {{- range $.Values.fsm.gateway.nodePorts }} + {{- if (eq .port $listenerPort) }} + nodePort: {{ .nodePort }} + {{- end }} + {{- end }} + {{- end }} {{- end }} {{- end }} selector: app: fsm-gateway - gateway.flomesh.io/ns: {{ .Values.gwy.metadata.namespace }} + gateway.flomesh.io/ns: {{ .Values.fsm.gateway.namespace }} + gateway.flomesh.io/name: {{ .Values.fsm.gateway.name }} {{- end }} \ No newline at end of file diff --git a/charts/gateway/templates/service-udp.yaml b/charts/gateway/templates/service-udp.yaml index 5923f5c1f..905825134 100644 --- a/charts/gateway/templates/service-udp.yaml +++ b/charts/gateway/templates/service-udp.yaml @@ -2,35 +2,46 @@ apiVersion: v1 kind: Service metadata: - name: {{ printf "fsm-gateway-%s-udp" .Values.gwy.metadata.namespace }} - namespace: {{ .Values.gwy.metadata.namespace }} - {{- if .Values.gwy.spec.infrastructure }} - {{- with .Values.gwy.spec.infrastructure.annotations }} + name: {{ printf "fsm-gateway-%s-%s-udp" .Values.fsm.gateway.namespace .Values.fsm.gateway.name }} + namespace: {{ .Values.fsm.gateway.namespace }} + {{- if .Values.fsm.gateway.infrastructure }} + {{- with .Values.fsm.gateway.infrastructure.annotations }} annotations: {{- toYaml . | nindent 4 }} {{- end }} {{- end }} labels: {{- include "fsm.labels" . | nindent 4 }} - {{- if .Values.gwy.spec.infrastructure }} - {{- with .Values.gwy.spec.infrastructure.labels }} + {{- if .Values.fsm.gateway.infrastructure }} + {{- with .Values.fsm.gateway.infrastructure.labels }} {{- toYaml . | nindent 4 }} {{- end }} {{- end }} app: fsm-gateway - gateway.flomesh.io/ns: {{ .Values.gwy.metadata.namespace }} + gateway.flomesh.io/ns: {{ .Values.fsm.gateway.namespace }} + gateway.flomesh.io/name: {{ .Values.fsm.gateway.name }} spec: - type: LoadBalancer + type: {{ .Values.fsm.gateway.serviceType }} + {{- $setNodePorts := (and (eq .Values.fsm.gateway.serviceType "NodePort") .Values.fsm.gateway.nodePorts)}} ports: - {{- range $listener := .Values.listeners }} - {{- if eq $listener.protocol "UDP" }} - - name: {{ $listener.name }} - port: {{ $listener.port }} - targetPort: {{ ternary (add 60000 $listener.port) $listener.port (lt (int $listener.port) 1024)}} + {{- range .Values.fsm.gateway.listeners }} + {{- if eq .protocol "UDP" }} + {{- $listenerPort := .port }} + - name: {{ .name }} + port: {{ $listenerPort }} + targetPort: {{ ternary (add 60000 $listenerPort) $listenerPort (lt (int $listenerPort) 1024)}} protocol: UDP + {{- if $setNodePorts }} + {{- range $.Values.fsm.gateway.nodePorts }} + {{- if (eq .port $listenerPort) }} + nodePort: {{ .nodePort }} + {{- end }} + {{- end }} + {{- end }} {{- end }} {{- end }} selector: app: fsm-gateway - gateway.flomesh.io/ns: {{ .Values.gwy.metadata.namespace }} + gateway.flomesh.io/ns: {{ .Values.fsm.gateway.namespace }} + gateway.flomesh.io/name: {{ .Values.fsm.gateway.name }} {{- end }} \ No newline at end of file diff --git a/charts/gateway/templates/serviceaccount.yaml b/charts/gateway/templates/serviceaccount.yaml index 3f3d69a5b..fcceacc6f 100644 --- a/charts/gateway/templates/serviceaccount.yaml +++ b/charts/gateway/templates/serviceaccount.yaml @@ -3,7 +3,7 @@ apiVersion: v1 kind: ServiceAccount metadata: name: {{ include "fsm.gateway.serviceAccountName" . }} - namespace: {{ .Values.gwy.metadata.namespace }} + namespace: {{ .Values.fsm.gateway.namespace }} labels: {{- include "fsm.labels" . | nindent 4 }} app: fsm-gateway diff --git a/charts/gateway/values.yaml b/charts/gateway/values.yaml index d7022fa65..f4f96dc50 100644 --- a/charts/gateway/values.yaml +++ b/charts/gateway/values.yaml @@ -30,8 +30,13 @@ fsm: # -- Namespace to deploy FSM in. If not specified, the Helm release namespace is used. fsmNamespace: "" - fsmGateway: + gateway: logLevel: info + + # -- FSM Gateway's service type, only LoadBalancer and NodePort are supported + serviceType: LoadBalancer + + # -- FSM Gateway's replica count replicas: 1 # -- FSM Gateway's container resource parameters. resources: @@ -171,4 +176,32 @@ fsm: memory: "512M" requests: cpu: "200m" - memory: "128M" \ No newline at end of file + memory: "128M" + + # -- NodePort service configuration + # nodePorts: + # - port: 80 + # nodePort: 30080 + # - port: 443 + # nodePort: 30443 + # - port: 53 + # nodePort: 30053 + nodePorts: [] + + # -- Gateway's namespace, not overridable by parameterRef + namespace: default + + # -- Gateway's name, not overridable by parameterRef + name: UNKNOWN + + # -- Gateway's infrastructure, override by gateway.spec.infrastructure.annotations and gateway.spec.infrastructure.labels + # -- NOT override by parameterRef + infrastructure: + annotations: {} + labels: {} + + # -- Gateway's listeners, not overridable by parameterRef + listeners: {} + + + diff --git a/cmd/fsm-gateway/fsm-gateway.go b/cmd/fsm-gateway/fsm-gateway.go index dca8c4f3a..efc255cc2 100644 --- a/cmd/fsm-gateway/fsm-gateway.go +++ b/cmd/fsm-gateway/fsm-gateway.go @@ -70,6 +70,8 @@ var ( fsmNamespace string fsmMeshConfigName string fsmVersion string + gatewayNamespace string + gatewayName string meta metadata ) @@ -80,6 +82,8 @@ func init() { flags.StringVar(&fsmNamespace, "fsm-namespace", "", "FSM controller's namespace") flags.StringVar(&fsmMeshConfigName, "fsm-config-name", "fsm-mesh-config", "Name of the FSM MeshConfig") flags.StringVar(&fsmVersion, "fsm-version", "", "Version of FSM") + flags.StringVar(&gatewayNamespace, "gateway-namespace", "", "Namespace of Gateway") + flags.StringVar(&gatewayName, "gateway-name", "", "Name of Gateway") meta = getMetadata() } @@ -155,7 +159,7 @@ func main() { func codebase(cfg configurator.Configurator) string { repoHost := fmt.Sprintf("%s.%s.svc", constants.FSMControllerName, fsmNamespace) repoPort := cfg.GetProxyServerPort() - return fmt.Sprintf("%s://%s:%d/repo%s/", "http", repoHost, repoPort, utils.GatewayCodebasePath(meta.PodNamespace)) + return fmt.Sprintf("%s://%s:%d/repo%s/", "http", repoHost, repoPort, utils.GatewayCodebasePath(gatewayNamespace, gatewayName)) } func calcPipySpawn(kubeClient kubernetes.Interface) int64 { diff --git a/docs/tests/gateway-api/README.md b/docs/tests/gateway-api/README.md index e3ee4fb49..ea075ffd2 100644 --- a/docs/tests/gateway-api/README.md +++ b/docs/tests/gateway-api/README.md @@ -74,6 +74,7 @@ kubectl create ns tcp-route kubectl create ns tcp kubectl create ns udp-route kubectl create ns udp +kubectl create ns nodeport kubectl label ns http-route app=http-cross kubectl label ns grpc-route app=grpc-cross @@ -138,7 +139,7 @@ metadata: data: values.yaml: | fsm: - fsmGateway: + gateway: replicas: 2 resources: requests: @@ -2437,4 +2438,53 @@ spec: kind: Service name: httpbin-cross EOF +``` + +### Test Gateway in NodePort mode + +- Create ConfigMap for configuring the Gateway +```shell +kubectl apply -f - < active gateway - activeGateways map[string]*gwv1.Gateway ) -type gatewayValues struct { - Gateway *gwv1.Gateway `json:"gwy,omitempty"` - Listeners []gwpkg.Listener `json:"listeners,omitempty"` +type listener struct { + Name gwv1.SectionName `json:"name"` + Port gwv1.PortNumber `json:"port"` + Protocol gwv1.ProtocolType `json:"protocol"` } type gatewayReconciler struct { - recorder record.EventRecorder - fctx *fctx.ControllerContext - gatewayAPIClient gwclient.Interface + recorder record.EventRecorder + fctx *fctx.ControllerContext } func (r *gatewayReconciler) NeedLeaderElection() bool { return true } -func init() { - activeGateways = make(map[string]*gwv1.Gateway) -} - // NewGatewayReconciler returns a new reconciler for Gateway resources func NewGatewayReconciler(ctx *fctx.ControllerContext) controllers.Reconciler { return &gatewayReconciler{ - recorder: ctx.Manager.GetEventRecorderFor("Gateway"), - fctx: ctx, - gatewayAPIClient: gatewayApiClientset.NewForConfigOrDie(ctx.KubeConfig), + recorder: ctx.Manager.GetEventRecorderFor("Gateway"), + fctx: ctx, } } @@ -136,198 +130,183 @@ func (r *gatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct return ctrl.Result{}, nil } - effectiveGatewayClass, err := r.findEffectiveGatewayClass(ctx) + gatewayClass, err := gwutils.FindGatewayClassByName(r.fctx.Manager.GetCache(), string(gateway.Spec.GatewayClassName)) if err != nil { + if errors.IsNotFound(err) { + log.Warn().Msgf("GatewayClass %s not found, ignore processing Gateway resource %s.", gateway.Spec.GatewayClassName, req.NamespacedName.String()) + return ctrl.Result{}, nil + } + return ctrl.Result{}, err } - if effectiveGatewayClass == nil { + if gatewayClass == nil { log.Warn().Msgf("No effective GatewayClass, ignore processing Gateway resource %s.", req.NamespacedName) return ctrl.Result{}, nil } - result, err := r.updateGatewayStatus(ctx, gateway, effectiveGatewayClass) - if err != nil { - return result, err - } + update := gw.NewGatewayStatusUpdate( + gateway, + &gateway.ObjectMeta, + &gateway.TypeMeta, + &gateway.Status, + ) - // 5. update listener status of this gateway no matter it's accepted or not - result, err = r.updateListenerStatus(ctx, gateway) - if err != nil { + if result, err := r.computeGatewayStatus(ctx, gateway, update); err != nil || result.RequeueAfter > 0 || result.Requeue { return result, err } - result, err = r.updateGatewayAddresses(ctx, gateway) - if err != nil || result.RequeueAfter > 0 || result.Requeue { - return result, err - } + r.fctx.StatusUpdater.Send(status.Update{ + Resource: &gwv1.Gateway{}, + NamespacedName: client.ObjectKeyFromObject(gateway), + Mutator: update, + }) r.fctx.GatewayEventHandler.OnAdd(gateway, false) return ctrl.Result{}, nil } -func (r *gatewayReconciler) findEffectiveGatewayClass(ctx context.Context) (*gwv1.GatewayClass, error) { - var gatewayClasses gwv1.GatewayClassList - if err := r.fctx.List(ctx, &gatewayClasses); err != nil { - return nil, fmt.Errorf("failed to list gateway classes: %s", err) +func (r *gatewayReconciler) computeGatewayStatus(ctx context.Context, gateway *gwv1.Gateway, update *gw.GatewayStatusUpdate) (ctrl.Result, error) { + // 1. compute listener status & accepted status + result, err := r.computeListenerStatus(ctx, gateway, update) + if err != nil { + return result, err } - var effectiveGatewayClass *gwv1.GatewayClass - for idx, cls := range gatewayClasses.Items { - cls := cls - if gwutils.IsEffectiveGatewayClass(&cls) { - effectiveGatewayClass = &gatewayClasses.Items[idx] - break - } + // 2. so far, it's accepted, just deploy it if not + if result, err := r.applyGateway(gateway, update); err != nil { + return result, err } - return effectiveGatewayClass, nil -} - -func (r *gatewayReconciler) updateGatewayStatus(ctx context.Context, gateway *gwv1.Gateway, effectiveGatewayClass *gwv1.GatewayClass) (ctrl.Result, error) { - // 1. List all Gateways in the namespace whose GatewayClass is current effective class - gatewayList := &gwv1.GatewayList{} - if err := r.fctx.List(ctx, gatewayList, client.InNamespace(gateway.Namespace)); err != nil { - log.Error().Msgf("Failed to list all gateways in namespace %s: %s", gateway.Namespace, err) - return ctrl.Result{}, err + // 3. compute gateway address and programmed status + result, err = r.updateGatewayAddresses(ctx, gateway, update) + if err != nil || result.RequeueAfter > 0 || result.Requeue { + return result, err } - // 2. Find the oldest Gateway in the namespace, if CreateTimestamp is equal, then sort by alphabet order asc. - // If spec.GatewayClassName equals effectiveGatewayClass then it's a valid gateway - // Otherwise, it's invalid - validGateways := make([]*gwv1.Gateway, 0) - invalidGateways := make([]*gwv1.Gateway, 0) + if !update.ConditionExists(gwv1.GatewayConditionAccepted) { + r.recorder.Eventf(gateway, corev1.EventTypeNormal, "Accepted", "Gateway is accepted") - for _, gw := range gatewayList.Items { - gw := gw // fix lint GO-LOOP-REF - if string(gw.Spec.GatewayClassName) == effectiveGatewayClass.Name { - validGateways = append(validGateways, &gw) - } else { - invalidGateways = append(invalidGateways, &gw) - } + update.AddCondition( + gwv1.GatewayConditionAccepted, + metav1.ConditionTrue, + gwv1.GatewayReasonAccepted, + "Gateway is accepted", + ) } - sort.Slice(validGateways, func(i, j int) bool { - if validGateways[i].CreationTimestamp.Time.Equal(validGateways[j].CreationTimestamp.Time) { - return client.ObjectKeyFromObject(validGateways[i]).String() < client.ObjectKeyFromObject(validGateways[j]).String() - } - - return validGateways[i].CreationTimestamp.Time.Before(validGateways[j].CreationTimestamp.Time) - }) + if !update.ConditionExists(gwv1.GatewayConditionProgrammed) { + r.recorder.Eventf(gateway, corev1.EventTypeNormal, "Programmed", "Gateway is programmed") - // 3. Set the oldest as Accepted and the rest are unaccepted - statusChangedGateways := make([]*gwv1.Gateway, 0) - for i := range validGateways { - if i == 0 { - if !gwutils.IsAcceptedGateway(validGateways[i]) { - r.setAccepted(validGateways[i]) - statusChangedGateways = append(statusChangedGateways, validGateways[i]) - } - } else { - if gwutils.IsAcceptedGateway(validGateways[i]) { - r.setUnaccepted(validGateways[i]) - statusChangedGateways = append(statusChangedGateways, validGateways[i]) - } - } - } - - // in case of effective GatewayClass changed or spec.GatewayClassName was changed - for i := range invalidGateways { - if gwutils.IsAcceptedGateway(invalidGateways[i]) { - r.setUnaccepted(invalidGateways[i]) - statusChangedGateways = append(statusChangedGateways, invalidGateways[i]) - } + update.AddCondition( + gwv1.GatewayConditionProgrammed, + metav1.ConditionTrue, + gwv1.GatewayReasonProgrammed, + "Gateway is programmed", + ) } - // 4. update status - for _, gw := range statusChangedGateways { - result, err := r.updateStatus(ctx, gw) - if err != nil { - return result, err - } - } return ctrl.Result{}, nil } -func (r *gatewayReconciler) updateGatewayAddresses(ctx context.Context, gateway *gwv1.Gateway) (ctrl.Result, error) { - // 6. after all status of gateways in the namespace have been updated successfully - // list all gateways in the namespace and deploy/redeploy the effective one - activeGateway, err := r.findActiveGatewayByNamespace(ctx, gateway.Namespace) - if err != nil { - return ctrl.Result{}, err - } - - if activeGateway != nil && !isSameGateway(activeGateways[gateway.Namespace], activeGateway) { - result, err := r.applyGateway(activeGateway) - if err != nil { - return result, err - } - - activeGateways[gateway.Namespace] = activeGateway - } - - if activeGateway == nil { - return ctrl.Result{}, nil - } - - // 7. update addresses of Gateway status if any IP is allocated - serviceName := lbServiceName(activeGateway) - if serviceName == "" { - log.Warn().Msgf("[GW] No supported service protocols for Gateway %s/%s, only TCP and UDP are supported now.", activeGateway.Namespace, activeGateway.Name) - return ctrl.Result{}, nil - } - - lbSvc := &corev1.Service{} - key := client.ObjectKey{ - Namespace: activeGateway.Namespace, - Name: serviceName, - } - if err := r.fctx.Get(ctx, key, lbSvc); err != nil { - return ctrl.Result{}, err - } +func (r *gatewayReconciler) computeListenerStatus(_ context.Context, gateway *gwv1.Gateway, update *gw.GatewayStatusUpdate) (ctrl.Result, error) { + invalidListeners := invalidateListeners(gateway.Spec.Listeners) + for name, cond := range invalidListeners { + update.AddListenerCondition( + string(name), + gwv1.ListenerConditionType(cond.Type), + cond.Status, + gwv1.ListenerConditionReason(cond.Reason), + cond.Message, + ) + } + + addInvalidListenerCondition := func(name gwv1.SectionName, msg string) { + update.AddListenerCondition( + string(name), + gwv1.ListenerConditionProgrammed, + metav1.ConditionFalse, + gwv1.ListenerReasonInvalid, + msg, + ) + } + + for _, listener := range gateway.Spec.Listeners { + groupKinds := supportedRouteGroupKinds(gateway, listener, update) + update.SetListenerSupportedKinds(string(listener.Name), groupKinds) + + if listener.AllowedRoutes != nil && listener.AllowedRoutes.Namespaces != nil && + listener.AllowedRoutes.Namespaces.From != nil && *listener.AllowedRoutes.Namespaces.From == gwv1.NamespacesFromSelector { + if listener.AllowedRoutes.Namespaces.Selector == nil { + addInvalidListenerCondition(listener.Name, "Listener.AllowedRoutes.Namespaces.Selector is required when Listener.AllowedRoutes.Namespaces.From is set to \"Selector\".") + continue + } - if lbSvc.Spec.Type != corev1.ServiceTypeLoadBalancer { - return ctrl.Result{}, nil - } + if len(listener.AllowedRoutes.Namespaces.Selector.MatchExpressions)+len(listener.AllowedRoutes.Namespaces.Selector.MatchLabels) == 0 { + addInvalidListenerCondition(listener.Name, "Listener.AllowedRoutes.Namespaces.Selector must specify at least one MatchLabel or MatchExpression.") + continue + } - if len(lbSvc.Status.LoadBalancer.Ingress) == 0 { - log.Debug().Msgf("[GW] No ingress IPs found for service %s/%s", lbSvc.Namespace, lbSvc.Name) - if len(activeGateway.Status.Addresses) == 0 { - log.Debug().Msgf("[GW] No addresses found for gateway %s/%s", activeGateway.Namespace, activeGateway.Name) - defer r.recorder.Eventf(activeGateway, corev1.EventTypeNormal, "UpdateAddresses", "Addresses of gateway has not been assigned yet") + var err error + _, err = metav1.LabelSelectorAsSelector(listener.AllowedRoutes.Namespaces.Selector) + if err != nil { + addInvalidListenerCondition(listener.Name, fmt.Sprintf("Error parsing Listener.AllowedRoutes.Namespaces.Selector: %v.", err)) + continue + } } - log.Debug().Msgf("[GW] Requeue gateway %s/%s after 3 second", activeGateway.Namespace, activeGateway.Name) - return ctrl.Result{RequeueAfter: 3 * time.Second}, nil - } - - addresses := gatewayAddresses(activeGateway, lbSvc) - if len(addresses) > 0 { - log.Debug().Msgf("[GW] Addresses of gateway %s/%s will be updated to: %s", activeGateway.Namespace, activeGateway.Name, strings.Join(addressesToStrings(addresses), ",")) - activeGateway.Status.Addresses = addresses - if err := r.fctx.Status().Update(ctx, activeGateway); err != nil { - //defer r.recorder.Eventf(activeGateway, corev1.EventTypeWarning, "UpdateAddresses", "Failed to update addresses of gateway: %s", err) - - return ctrl.Result{}, err + if _, ok := invalidListeners[listener.Name]; ok { + continue } - } - defer r.recorder.Eventf(activeGateway, corev1.EventTypeNormal, "UpdateAddresses", "Addresses of gateway is updated: %s", strings.Join(addressesToStrings(addresses), ",")) + listenerStatus := update.GetListenerStatus(string(listener.Name)) + + if listenerStatus == nil || len(listenerStatus.Conditions) == 0 { + update.AddListenerCondition( + string(listener.Name), + gwv1.ListenerConditionProgrammed, + metav1.ConditionTrue, + gwv1.ListenerReasonProgrammed, + "Valid listener", + ) + update.AddListenerCondition( + string(listener.Name), + gwv1.ListenerConditionAccepted, + metav1.ConditionTrue, + gwv1.ListenerReasonAccepted, + "Listener accepted", + ) + update.AddListenerCondition( + string(listener.Name), + gwv1.ListenerConditionResolvedRefs, + metav1.ConditionTrue, + gwv1.ListenerReasonResolvedRefs, + "Listener references resolved", + ) + } else { + if metautil.FindStatusCondition(listenerStatus.Conditions, string(gwv1.ListenerConditionProgrammed)) == nil { + addInvalidListenerCondition(listener.Name, "Invalid listener, see other listener conditions for details") + } - // if there's any previous active gateways and has been assigned addresses, clean it up - gatewayList := &gwv1.GatewayList{} - if err := r.fctx.List(ctx, gatewayList, client.InNamespace(activeGateway.Namespace)); err != nil { - log.Error().Msgf("Failed to list all gateways in namespace %s: %s", activeGateway.Namespace, err) - return ctrl.Result{}, err - } + if metautil.FindStatusCondition(listenerStatus.Conditions, string(gwv1.ListenerConditionAccepted)) == nil { + update.AddListenerCondition( + string(listener.Name), + gwv1.ListenerConditionAccepted, + metav1.ConditionTrue, + gwv1.ListenerReasonAccepted, + "Listener accepted", + ) + } - for _, gw := range gatewayList.Items { - gw := gw // fix lint GO-LOOP-REF - if gw.Name != activeGateway.Name && len(gw.Status.Addresses) > 0 { - gw.Status.Addresses = nil - if err := r.fctx.Status().Update(ctx, &gw); err != nil { - return ctrl.Result{}, err + if metautil.FindStatusCondition(listenerStatus.Conditions, string(gwv1.ListenerConditionResolvedRefs)) == nil { + update.AddListenerCondition( + string(listener.Name), + gwv1.ListenerConditionResolvedRefs, + metav1.ConditionTrue, + gwv1.ListenerReasonResolvedRefs, + "Listener references resolved", + ) } } } @@ -335,121 +314,86 @@ func (r *gatewayReconciler) updateGatewayAddresses(ctx context.Context, gateway return ctrl.Result{}, nil } -func lbServiceName(activeGateway *gwv1.Gateway) string { - if hasTCP(activeGateway) { - return fmt.Sprintf("fsm-gateway-%s-tcp", activeGateway.Namespace) - } - - if hasUDP(activeGateway) { - return fmt.Sprintf("fsm-gateway-%s-udp", activeGateway.Namespace) +func invalidateListeners(listeners []gwv1.Listener) map[gwv1.SectionName]metav1.Condition { + conflictCondition := func(msg string) metav1.Condition { + return metav1.Condition{ + Type: string(gwv1.ListenerConditionConflicted), + Status: metav1.ConditionTrue, + Reason: string(gwv1.ListenerReasonHostnameConflict), + Message: msg, + } } - return "" -} - -func (r *gatewayReconciler) updateListenerStatus(ctx context.Context, gateway *gwv1.Gateway) (ctrl.Result, error) { - if len(gateway.Annotations) == 0 { - gateway.Annotations = make(map[string]string) - } + invalidListenerConditions := map[gwv1.SectionName]metav1.Condition{} - oldHash := gateway.Annotations[constants.GatewayListenersHashAnnotation] - hash := utils.SimpleHash(gateway.Spec.Listeners) + for i, listener := range listeners { + // Check for a valid hostname. + if hostname := ptr.Deref(listener.Hostname, ""); len(hostname) > 0 { + if err := isValidHostname(string(hostname)); err != nil { + invalidListenerConditions[listener.Name] = metav1.Condition{ + Type: string(gwv1.ListenerConditionProgrammed), + Status: metav1.ConditionFalse, + Reason: string(gwv1.ListenerReasonInvalid), + Message: fmt.Sprintf("Invalid hostname %q: %v", hostname, err), + } + continue + } + } - if oldHash != hash { - gateway.Annotations[constants.GatewayListenersHashAnnotation] = hash - if err := r.fctx.Update(ctx, gateway); err != nil { - return ctrl.Result{}, err + // Check for a supported protocol. + switch listener.Protocol { + case gwv1.HTTPProtocolType, gwv1.HTTPSProtocolType, gwv1.TLSProtocolType, gwv1.TCPProtocolType, gwv1.UDPProtocolType: + default: + invalidListenerConditions[listener.Name] = conflictCondition(fmt.Sprintf("Listener protocol %q is unsupported, must be one of HTTP, HTTPS, TLS, TCP or UDP", listener.Protocol)) + continue } - existingListenerStatus := make(map[gwv1.SectionName]gwv1.ListenerStatus) - for _, status := range gateway.Status.Listeners { - existingListenerStatus[status.Name] = status + if listener.Port > 60000 && listener.Port <= 65535 { + invalidListenerConditions[listener.Name] = conflictCondition(fmt.Sprintf("Listener port %d is invalid, must be in the range 1-60000", listener.Port)) + continue } - gateway.Status.Listeners = nil - listenerStatus := make([]gwv1.ListenerStatus, 0) - for _, listener := range gateway.Spec.Listeners { - status, ok := existingListenerStatus[listener.Name] - if ok { - // update existing status - programmedConditionExists := false - acceptedConditionExists := false - for _, cond := range status.Conditions { - if cond.Type == string(gwv1.ListenerConditionProgrammed) { - programmedConditionExists = true - } - if cond.Type == string(gwv1.ListenerConditionAccepted) { - acceptedConditionExists = true - } - } + func() { + for j := range i { + otherListener := listeners[j] - if !programmedConditionExists { - metautil.SetStatusCondition(&status.Conditions, metav1.Condition{ - Type: string(gwv1.ListenerConditionProgrammed), - Status: metav1.ConditionFalse, - ObservedGeneration: gateway.Generation, - LastTransitionTime: metav1.Time{Time: time.Now()}, - Reason: string(gwv1.ListenerReasonInvalid), - Message: fmt.Sprintf("Invalid listener %q[:%d]", listener.Name, listener.Port), - }) + if listener.Port != otherListener.Port { + continue } - if !acceptedConditionExists { - metautil.SetStatusCondition(&status.Conditions, metav1.Condition{ - Type: string(gwv1.ListenerConditionAccepted), - Status: metav1.ConditionTrue, - ObservedGeneration: gateway.Generation, - LastTransitionTime: metav1.Time{Time: time.Now()}, - Reason: string(gwv1.ListenerReasonAccepted), - Message: fmt.Sprintf("listener %q[:%d] is accepted.", listener.Name, listener.Port), - }) - } - } else { - // create new status - status = gwv1.ListenerStatus{Name: listener.Name} - kinds, conditions := supportedRouteGroupKinds(gateway, listener) - - if len(conditions) == 0 { - status.Conditions = []metav1.Condition{ - { - Type: string(gwv1.ListenerConditionAccepted), - Status: metav1.ConditionTrue, - ObservedGeneration: gateway.Generation, - LastTransitionTime: metav1.Time{Time: time.Now()}, - Reason: string(gwv1.ListenerReasonAccepted), - Message: fmt.Sprintf("listener %q[:%d] is accepted.", listener.Name, listener.Port), - }, - { - Type: string(gwv1.ListenerConditionProgrammed), - Status: metav1.ConditionTrue, - ObservedGeneration: gateway.Generation, - LastTransitionTime: metav1.Time{Time: time.Now()}, - Reason: string(gwv1.ListenerReasonProgrammed), - Message: fmt.Sprintf("Valid listener %q[:%d]", listener.Name, listener.Port), - }, + if listener.Protocol != otherListener.Protocol { + // same port, different protocol, not allowed + invalidListenerConditions[listener.Name] = metav1.Condition{ + Type: string(gwv1.ListenerConditionConflicted), + Status: metav1.ConditionTrue, + Reason: string(gwv1.ListenerReasonProtocolConflict), + Message: "All Listener protocols for a given port must be the same", } - } else { - status.Conditions = conditions - } - - status.SupportedKinds = kinds - } - listenerStatus = append(listenerStatus, status) - } + return + } - if len(listenerStatus) > 0 { - gateway.Status.Listeners = listenerStatus - if err := r.fctx.Status().Update(ctx, gateway); err != nil { - return ctrl.Result{}, err + switch listener.Protocol { + case gwv1.HTTPProtocolType, gwv1.HTTPSProtocolType, gwv1.TLSProtocolType: + // Hostname conflict + if ptr.Deref(listener.Hostname, "") == ptr.Deref(otherListener.Hostname, "") { + invalidListenerConditions[listener.Name] = metav1.Condition{ + Type: string(gwv1.ListenerConditionConflicted), + Status: metav1.ConditionTrue, + Reason: string(gwv1.ListenerReasonHostnameConflict), + Message: "All Listener hostnames for a given port must be unique", + } + return + } + } } - } + }() } - return ctrl.Result{}, nil + return invalidListenerConditions } -func supportedRouteGroupKinds(gateway *gwv1.Gateway, listener gwv1.Listener) ([]gwv1.RouteGroupKind, []metav1.Condition) { +func supportedRouteGroupKinds(_ *gwv1.Gateway, listener gwv1.Listener, update *gw.GatewayStatusUpdate) []gwv1.RouteGroupKind { if len(listener.AllowedRoutes.Kinds) == 0 { switch listener.Protocol { case gwv1.HTTPProtocolType, gwv1.HTTPSProtocolType: @@ -462,7 +406,7 @@ func supportedRouteGroupKinds(gateway *gwv1.Gateway, listener gwv1.Listener) ([] Group: gwutils.GroupPointer(constants.GatewayAPIGroup), Kind: constants.GatewayAPIGRPCRouteKind, }, - }, nil + } case gwv1.TLSProtocolType: return []gwv1.RouteGroupKind{ { @@ -473,37 +417,36 @@ func supportedRouteGroupKinds(gateway *gwv1.Gateway, listener gwv1.Listener) ([] Group: gwutils.GroupPointer(constants.GatewayAPIGroup), Kind: constants.GatewayAPITCPRouteKind, }, - }, nil + } case gwv1.TCPProtocolType: return []gwv1.RouteGroupKind{ { Group: gwutils.GroupPointer(constants.GatewayAPIGroup), Kind: constants.GatewayAPITCPRouteKind, }, - }, nil + } case gwv1.UDPProtocolType: return []gwv1.RouteGroupKind{ { Group: gwutils.GroupPointer(constants.GatewayAPIGroup), Kind: constants.GatewayAPIUDPRouteKind, }, - }, nil + } } } kinds := make([]gwv1.RouteGroupKind, 0) - conditions := make([]metav1.Condition, 0) for _, routeKind := range listener.AllowedRoutes.Kinds { if routeKind.Group != nil && *routeKind.Group != constants.GatewayAPIGroup { - conditions = append(conditions, metav1.Condition{ - Type: string(gwv1.ListenerConditionResolvedRefs), - Status: metav1.ConditionFalse, - ObservedGeneration: gateway.Generation, - LastTransitionTime: metav1.Time{Time: time.Now()}, - Reason: string(gwv1.ListenerReasonInvalidRouteKinds), - Message: fmt.Sprintf("Group %q is not supported, group must be %q", *routeKind.Group, gwv1.GroupName), - }) + update.AddListenerCondition( + string(listener.Name), + gwv1.ListenerConditionResolvedRefs, + metav1.ConditionFalse, + gwv1.ListenerReasonInvalidRouteKinds, + fmt.Sprintf("Group %q is not supported, group must be %q", *routeKind.Group, gwv1.GroupName), + ) + continue } @@ -512,74 +455,68 @@ func supportedRouteGroupKinds(gateway *gwv1.Gateway, listener gwv1.Listener) ([] routeKind.Kind != constants.GatewayAPIGRPCRouteKind && routeKind.Kind != constants.GatewayAPITCPRouteKind && routeKind.Kind != constants.GatewayAPIUDPRouteKind { - conditions = append(conditions, metav1.Condition{ - Type: string(gwv1.ListenerConditionResolvedRefs), - Status: metav1.ConditionFalse, - ObservedGeneration: gateway.Generation, - LastTransitionTime: metav1.Time{Time: time.Now()}, - Reason: string(gwv1.ListenerReasonInvalidRouteKinds), - Message: fmt.Sprintf("Kind %q is not supported, kind must be %q, %q, %q, %q or %q", routeKind.Kind, constants.GatewayAPIHTTPRouteKind, constants.GatewayAPIGRPCRouteKind, constants.GatewayAPITLSRouteKind, constants.GatewayAPITCPRouteKind, constants.GatewayAPIUDPRouteKind), - }) + update.AddListenerCondition( + string(listener.Name), + gwv1.ListenerConditionResolvedRefs, + metav1.ConditionFalse, + gwv1.ListenerReasonInvalidRouteKinds, + fmt.Sprintf("Kind %q is not supported, kind must be %q, %q, %q, %q or %q", routeKind.Kind, constants.GatewayAPIHTTPRouteKind, constants.GatewayAPIGRPCRouteKind, constants.GatewayAPITLSRouteKind, constants.GatewayAPITCPRouteKind, constants.GatewayAPIUDPRouteKind), + ) continue } if routeKind.Kind == constants.GatewayAPIHTTPRouteKind && listener.Protocol != gwv1.HTTPProtocolType && listener.Protocol != gwv1.HTTPSProtocolType { - conditions = append(conditions, metav1.Condition{ - Type: string(gwv1.ListenerConditionResolvedRefs), - Status: metav1.ConditionFalse, - ObservedGeneration: gateway.Generation, - LastTransitionTime: metav1.Time{Time: time.Now()}, - Reason: string(gwv1.ListenerReasonInvalidRouteKinds), - Message: fmt.Sprintf("HTTPRoutes are incompatible with listener protocol %q", listener.Protocol), - }) + update.AddListenerCondition( + string(listener.Name), + gwv1.ListenerConditionResolvedRefs, + metav1.ConditionFalse, + gwv1.ListenerReasonInvalidRouteKinds, + fmt.Sprintf("HTTPRoutes are incompatible with listener protocol %q", listener.Protocol), + ) continue } if routeKind.Kind == constants.GatewayAPIGRPCRouteKind && listener.Protocol != gwv1.HTTPProtocolType && listener.Protocol != gwv1.HTTPSProtocolType { - conditions = append(conditions, metav1.Condition{ - Type: string(gwv1.ListenerConditionResolvedRefs), - Status: metav1.ConditionFalse, - ObservedGeneration: gateway.Generation, - LastTransitionTime: metav1.Time{Time: time.Now()}, - Reason: string(gwv1.ListenerReasonInvalidRouteKinds), - Message: fmt.Sprintf("GRPCRoutes are incompatible with listener protocol %q", listener.Protocol), - }) + update.AddListenerCondition( + string(listener.Name), + gwv1.ListenerConditionResolvedRefs, + metav1.ConditionFalse, + gwv1.ListenerReasonInvalidRouteKinds, + fmt.Sprintf("GRPCRoutes are incompatible with listener protocol %q", listener.Protocol), + ) continue } if routeKind.Kind == constants.GatewayAPITLSRouteKind && listener.Protocol != gwv1.TLSProtocolType { - conditions = append(conditions, metav1.Condition{ - Type: string(gwv1.ListenerConditionResolvedRefs), - Status: metav1.ConditionFalse, - ObservedGeneration: gateway.Generation, - LastTransitionTime: metav1.Time{Time: time.Now()}, - Reason: string(gwv1.ListenerReasonInvalidRouteKinds), - Message: fmt.Sprintf("TLSRoutes are incompatible with listener protocol %q", listener.Protocol), - }) + update.AddListenerCondition( + string(listener.Name), + gwv1.ListenerConditionResolvedRefs, + metav1.ConditionFalse, + gwv1.ListenerReasonInvalidRouteKinds, + fmt.Sprintf("TLSRoutes are incompatible with listener protocol %q", listener.Protocol), + ) continue } if routeKind.Kind == constants.GatewayAPITCPRouteKind && listener.Protocol != gwv1.TCPProtocolType && listener.Protocol != gwv1.TLSProtocolType { - conditions = append(conditions, metav1.Condition{ - Type: string(gwv1.ListenerConditionResolvedRefs), - Status: metav1.ConditionFalse, - ObservedGeneration: gateway.Generation, - LastTransitionTime: metav1.Time{Time: time.Now()}, - Reason: string(gwv1.ListenerReasonInvalidRouteKinds), - Message: fmt.Sprintf("TCPRoutes are incompatible with listener protocol %q", listener.Protocol), - }) + update.AddListenerCondition( + string(listener.Name), + gwv1.ListenerConditionResolvedRefs, + metav1.ConditionFalse, + gwv1.ListenerReasonInvalidRouteKinds, + fmt.Sprintf("TCPRoutes are incompatible with listener protocol %q", listener.Protocol), + ) continue } if routeKind.Kind == constants.GatewayAPIUDPRouteKind && listener.Protocol != gwv1.UDPProtocolType { - conditions = append(conditions, metav1.Condition{ - Type: string(gwv1.ListenerConditionResolvedRefs), - Status: metav1.ConditionFalse, - ObservedGeneration: gateway.Generation, - LastTransitionTime: metav1.Time{Time: time.Now()}, - Reason: string(gwv1.ListenerReasonInvalidRouteKinds), - Message: fmt.Sprintf("UDPRoutes are incompatible with listener protocol %q", listener.Protocol), - }) + update.AddListenerCondition( + string(listener.Name), + gwv1.ListenerConditionResolvedRefs, + metav1.ConditionFalse, + gwv1.ListenerReasonInvalidRouteKinds, + fmt.Sprintf("UDPRoutes are incompatible with listener protocol %q", listener.Protocol), + ) continue } @@ -589,195 +526,380 @@ func supportedRouteGroupKinds(gateway *gwv1.Gateway, listener gwv1.Listener) ([] }) } - return kinds, conditions + return kinds } -func gatewayAddresses(activeGateway *gwv1.Gateway, lbSvc *corev1.Service) []gwv1.GatewayStatusAddress { - existingIPs := gatewayIPs(activeGateway) - expectedIPs := lbIPs(lbSvc) - existingHostnames := gatewayHostnames(activeGateway) - expectedHostnames := lbHostnames(lbSvc) +func (r *gatewayReconciler) computeGatewayProgrammedCondition(ctx context.Context, gw *gwv1.Gateway, addresses []gwv1.GatewayStatusAddress, update *gw.GatewayStatusUpdate) bool { + if len(addresses) == 0 { + defer r.recorder.Eventf(gw, corev1.EventTypeWarning, "Addresses", "No addresses have been assigned to the Gateway") + + update.AddCondition( + gwv1.GatewayConditionProgrammed, + metav1.ConditionFalse, + gwv1.GatewayReasonAddressNotAssigned, + "No addresses have been assigned to the Gateway", + ) + return false + } + + //isSpecAddressAssigned := func(specAddresses []gwv1.GatewayAddress, statusAddresses []gwv1.GatewayStatusAddress) bool { + // if len(specAddresses) == 0 { + // return true + // } + // + // for _, specAddress := range specAddresses { + // for _, statusAddress := range statusAddresses { + // // Types must match + // if ptr.Deref(specAddress.Type, gwv1.IPAddressType) != ptr.Deref(statusAddress.Type, gwv1.IPAddressType) { + // continue + // } + // + // // Values must match + // if specAddress.Value != statusAddress.Value { + // continue + // } + // + // return true + // } + // } + // + // return false + //} + //if !isSpecAddressAssigned(gw.Spec.Addresses, addresses) { + // defer r.recorder.Eventf(gw, corev1.EventTypeWarning, "Addresses", "None of the addresses in Spec.Addresses have been assigned to the Gateway") + // + // return gatewayAddressNotAssignedCondition(gw, "None of the addresses in Spec.Addresses have been assigned to the Gateway"), false + //} + + deployment := r.gatewayDeployment(ctx, gw) + if deployment == nil || deployment.Status.AvailableReplicas == 0 { + defer r.recorder.Eventf(gw, corev1.EventTypeWarning, "Unavailable", "Gateway Deployment replicas unavailable") + + update.AddCondition( + gwv1.GatewayConditionProgrammed, + metav1.ConditionFalse, + gwv1.GatewayReasonNoResources, + "Deployment replicas unavailable", + ) + + return false + } + + defer r.recorder.Eventf(gw, corev1.EventTypeNormal, "Programmed", fmt.Sprintf("Address assigned to the Gateway, %d/%d Deployment replicas available", deployment.Status.AvailableReplicas, deployment.Status.Replicas)) + + update.AddCondition( + gwv1.GatewayConditionProgrammed, + metav1.ConditionTrue, + gwv1.GatewayReasonProgrammed, + fmt.Sprintf("Address assigned to the Gateway, %d/%d Deployment replicas available", deployment.Status.AvailableReplicas, deployment.Status.Replicas), + ) - sort.Strings(expectedIPs) - sort.Strings(existingIPs) - sort.Strings(existingHostnames) - sort.Strings(expectedHostnames) + return true +} - ipChanged := !utils.StringsEqual(expectedIPs, existingIPs) - hostnameChanged := !utils.StringsEqual(expectedHostnames, existingHostnames) - if !ipChanged && !hostnameChanged { - return nil +func (r *gatewayReconciler) updateGatewayAddresses(ctx context.Context, gateway *gwv1.Gateway, update *gw.GatewayStatusUpdate) (ctrl.Result, error) { + addresses := r.gatewayAddresses(ctx, gateway) + programmed := r.computeGatewayProgrammedCondition(ctx, gateway, addresses, update) + + if !programmed { + log.Debug().Msgf("[GW] Requeue gateway %s/%s after 5 second", gateway.Namespace, gateway.Name) + return ctrl.Result{RequeueAfter: 5 * time.Second}, nil } - addresses := make([]gwv1.GatewayStatusAddress, 0) - if ipChanged { - for _, ip := range expectedIPs { - addresses = append(addresses, gwv1.GatewayStatusAddress{ - Type: addressTypePointer(gwv1.IPAddressType), - Value: ip, - }) + update.SetAddresses(addresses) + + allListenersProgrammed := func(gw *gwv1.Gateway) bool { + for _, listener := range gw.Status.Listeners { + if !gwutils.IsListenerProgrammed(listener) { + return false + } } + + return true } - if hostnameChanged { - for _, hostname := range expectedHostnames { - addresses = append(addresses, gwv1.GatewayStatusAddress{ - Type: addressTypePointer(gwv1.HostnameAddressType), - Value: hostname, - }) - } + + if !allListenersProgrammed(gateway) { + defer r.recorder.Eventf(gateway, corev1.EventTypeWarning, "Listeners", "Not All listeners are programmed") + + update.AddCondition( + gwv1.GatewayConditionAccepted, + metav1.ConditionFalse, + gwv1.GatewayReasonListenersNotValid, + "Not all listeners are programmed", + ) } - return addresses + return ctrl.Result{}, nil } -func (r *gatewayReconciler) updateStatus(ctx context.Context, gw *gwv1.Gateway) (ctrl.Result, error) { - if err := r.fctx.Status().Update(ctx, gw); err != nil { - defer r.recorder.Eventf(gw, corev1.EventTypeWarning, "UpdateStatus", "Failed to update status of gateway: %s", err) - return ctrl.Result{}, err +func (r *gatewayReconciler) gatewayService(ctx context.Context, gateway *gwv1.Gateway) (*corev1.Service, error) { + gatewayServiceName := func(activeGateway *gwv1.Gateway) string { + if hasTCP(activeGateway) { + return fmt.Sprintf("fsm-gateway-%s-%s-tcp", activeGateway.Namespace, activeGateway.Name) + } + + if hasUDP(activeGateway) { + return fmt.Sprintf("fsm-gateway-%s-%s-udp", activeGateway.Namespace, activeGateway.Name) + } + + return "" } - if gwutils.IsAcceptedGateway(gw) { - defer r.recorder.Eventf(gw, corev1.EventTypeNormal, "Accepted", "Gateway is accepted") - } else { - defer r.recorder.Eventf(gw, corev1.EventTypeNormal, "Rejected", "Gateway in unaccepted due to it's not the oldest in namespace %s or its gatewayClassName is incorrect", gw.Namespace) + serviceName := gatewayServiceName(gateway) + if serviceName == "" { + log.Warn().Msgf("[GW] No supported service protocols for Gateway %s/%s, only TCP and UDP are supported now.", gateway.Namespace, gateway.Name) + return nil, fmt.Errorf("no supported service protocols for Gateway %s/%s, only TCP and UDP are supported", gateway.Namespace, gateway.Name) } - return ctrl.Result{}, nil + svc := &corev1.Service{} + key := client.ObjectKey{ + Namespace: gateway.Namespace, + Name: serviceName, + } + if err := r.fctx.Get(ctx, key, svc); err != nil { + return nil, err + } + + return svc, nil } -func gatewayIPs(gateway *gwv1.Gateway) []string { - var ips []string +func (r *gatewayReconciler) gatewayAddresses(ctx context.Context, gw *gwv1.Gateway) []gwv1.GatewayStatusAddress { + gwSvc, err := r.gatewayService(ctx, gw) + if err != nil { + log.Error().Msgf("Failed to get gateway service: %s", err) + return nil + } + + var addresses, hostnames []string - for _, addr := range gateway.Status.Addresses { - if addr.Type == addressTypePointer(gwv1.IPAddressType) && addr.Value != "" { - ips = append(ips, addr.Value) + switch gwSvc.Spec.Type { + case corev1.ServiceTypeLoadBalancer: + for i := range gwSvc.Status.LoadBalancer.Ingress { + switch { + case len(gwSvc.Status.LoadBalancer.Ingress[i].IP) > 0: + addresses = append(addresses, gwSvc.Status.LoadBalancer.Ingress[i].IP) + case len(gwSvc.Status.LoadBalancer.Ingress[i].Hostname) > 0: + if gwSvc.Status.LoadBalancer.Ingress[i].Hostname == "localhost" { + addresses = append(addresses, "127.0.0.1") + } + hostnames = append(hostnames, gwSvc.Status.LoadBalancer.Ingress[i].Hostname) + } } + case corev1.ServiceTypeNodePort: + addresses = append(addresses, r.getNodeIPs(ctx, gwSvc)...) + default: + return nil } - return ips -} - -func gatewayHostnames(gateway *gwv1.Gateway) []string { - var hostnames []string + var gwAddresses []gwv1.GatewayStatusAddress + for i := range addresses { + addr := gwv1.GatewayStatusAddress{ + Type: ptr.To(gwv1.IPAddressType), + Value: addresses[i], + } + gwAddresses = append(gwAddresses, addr) + } - for _, addr := range gateway.Status.Addresses { - if addr.Type == addressTypePointer(gwv1.HostnameAddressType) && addr.Value != "" { - hostnames = append(hostnames, addr.Value) + for i := range hostnames { + addr := gwv1.GatewayStatusAddress{ + Type: ptr.To(gwv1.HostnameAddressType), + Value: hostnames[i], } + gwAddresses = append(gwAddresses, addr) } - return hostnames + return gwAddresses } -func lbIPs(svc *corev1.Service) []string { - var ips []string +func (r *gatewayReconciler) getNodeIPs(ctx context.Context, svc *corev1.Service) []string { + pods := &corev1.PodList{} + if err := r.fctx.List( + ctx, + pods, + client.InNamespace(svc.Namespace), + client.MatchingLabelsSelector{ + Selector: labels.SelectorFromSet(svc.Spec.Selector), + }, + ); err != nil { + log.Error().Msgf("Failed to get pods: %s", err) + return nil + } - for _, ingress := range svc.Status.LoadBalancer.Ingress { - if ingress.IP != "" { - ips = append(ips, ingress.IP) + extIPs := sets.New[string]() + intIPs := sets.New[string]() + + for _, pod := range pods.Items { + if pod.Spec.NodeName == "" || pod.Status.PodIP == "" { + continue } - } - return ips -} + if !utils.IsPodStatusConditionTrue(pod.Status.Conditions, corev1.PodReady) { + continue + } + + node := &corev1.Node{} + if err := r.fctx.Get(ctx, client.ObjectKey{Name: pod.Spec.NodeName}, node); err != nil { + if errors.IsNotFound(err) { + continue + } -func lbHostnames(svc *corev1.Service) []string { - var hostnames []string + log.Error().Msgf("Failed to get node %q: %s", pod.Spec.NodeName, err) + return nil + } - for _, ingress := range svc.Status.LoadBalancer.Ingress { - if ingress.Hostname != "" { - hostnames = append(hostnames, ingress.Hostname) + for _, addr := range node.Status.Addresses { + switch addr.Type { + case corev1.NodeExternalIP: + extIPs.Insert(addr.Address) + case corev1.NodeInternalIP: + intIPs.Insert(addr.Address) + default: + continue + } } } - return hostnames -} + var nodeIPs []string + if len(extIPs) > 0 { + nodeIPs = extIPs.UnsortedList() + } else { + nodeIPs = intIPs.UnsortedList() + } -func addressTypePointer(addrType gwv1.AddressType) *gwv1.AddressType { - return &addrType -} + if version.IsDualStackEnabled(r.fctx.KubeClient) { + ips, err := utils.FilterByIPFamily(nodeIPs, svc) + if err != nil { + return nil + } -func addressesToStrings(addresses []gwv1.GatewayStatusAddress) []string { - result := make([]string, 0) - for _, addr := range addresses { - result = append(result, addr.Value) + nodeIPs = ips } - return result + return nodeIPs } -func (r *gatewayReconciler) findActiveGatewayByNamespace(ctx context.Context, namespace string) (*gwv1.Gateway, error) { - gatewayList := &gwv1.GatewayList{} - if err := r.fctx.List(ctx, gatewayList, client.InNamespace(namespace)); err != nil { - log.Error().Msgf("Failed to list all gateways in namespace %s: %s", namespace, err) - return nil, err +func (r *gatewayReconciler) gatewayDeployment(ctx context.Context, gw *gwv1.Gateway) *appsv1.Deployment { + deployment := &appsv1.Deployment{} + key := types.NamespacedName{ + Namespace: gw.Namespace, + Name: fmt.Sprintf("fsm-gateway-%s-%s", gw.Namespace, gw.Name), } - for _, gw := range gatewayList.Items { - gw := gw // fix lint GO-LOOP-REF - if gwutils.IsActiveGateway(&gw) { - return &gw, nil + if err := r.fctx.Get(ctx, key, deployment); err != nil { + if errors.IsNotFound(err) { + log.Warn().Msgf("Deployment %s not found", key.String()) + return nil } + + log.Error().Msgf("Failed to get deployment %s: %s", key.String(), err) + return nil } - return nil, nil + return deployment } -func isSameGateway(oldGateway, newGateway *gwv1.Gateway) bool { - return equality.Semantic.DeepEqual(oldGateway, newGateway) -} +//func isSameGateway(oldGateway, newGateway *gwv1.Gateway) bool { +// return equality.Semantic.DeepEqual(oldGateway, newGateway) +//} + +func (r *gatewayReconciler) applyGateway(gateway *gwv1.Gateway, update *gw.GatewayStatusUpdate) (ctrl.Result, error) { + if len(gateway.Spec.Addresses) > 0 { + update.AddCondition( + gwv1.GatewayConditionProgrammed, + metav1.ConditionFalse, + gwv1.GatewayReasonAddressNotAssigned, + ".spec.addresses is not supported yet.", + ) + + update.AddCondition( + gwv1.GatewayConditionAccepted, + metav1.ConditionFalse, + gwv1.GatewayReasonUnsupportedAddress, + ".spec.addresses is not supported yet.", + ) + + r.recorder.Eventf(gateway, corev1.EventTypeWarning, "Address", ".spec.addresses is not supported yet.") + + return ctrl.Result{}, nil + } -func (r *gatewayReconciler) applyGateway(gateway *gwv1.Gateway) (ctrl.Result, error) { mc := r.fctx.Configurator result, err := r.deriveCodebases(gateway, mc) if err != nil { - defer r.recorder.Eventf(gateway, corev1.EventTypeWarning, "DeriveCodebase", "Failed to derive codebase of gateway: %s", err) - return result, err } - defer r.recorder.Eventf(gateway, corev1.EventTypeNormal, "DeriveCodebase", "Derive codebase of gateway successfully") result, err = r.updateConfig(gateway, mc) if err != nil { - defer r.recorder.Eventf(gateway, corev1.EventTypeWarning, "UpdateRepo", "Failed to update repo config of gateway: %s", err) + return result, err + } + result, err = r.deployGateway(gateway, mc, update) + if err != nil { return result, err } - defer r.recorder.Eventf(gateway, corev1.EventTypeNormal, "UpdateRepo", "Update repo config of gateway successfully") - return r.deployGateway(gateway, mc) + //activeGateways[gateway.Namespace] = gateway + + return ctrl.Result{}, nil } func (r *gatewayReconciler) deriveCodebases(gw *gwv1.Gateway, _ configurator.Configurator) (ctrl.Result, error) { - gwPath := utils.GatewayCodebasePath(gw.Namespace) + gwPath := utils.GatewayCodebasePath(gw.Namespace, gw.Name) parentPath := utils.GetDefaultGatewaysPath() if err := r.fctx.RepoClient.DeriveCodebase(gwPath, parentPath); err != nil { + defer r.recorder.Eventf(gw, corev1.EventTypeWarning, "Codebase", "Failed to derive codebase of gateway: %s", err) + return ctrl.Result{RequeueAfter: 1 * time.Second}, err } return ctrl.Result{}, nil } -func (r *gatewayReconciler) updateConfig(_ *gwv1.Gateway, _ configurator.Configurator) (ctrl.Result, error) { +func (r *gatewayReconciler) updateConfig(gw *gwv1.Gateway, _ configurator.Configurator) (ctrl.Result, error) { // TODO: update pipy repo + // defer r.recorder.Eventf(gw, corev1.EventTypeWarning, "Repo", "Failed to update repo config of gateway: %s", err) + //defer r.recorder.Eventf(gw, corev1.EventTypeNormal, "Repo", "Update repo config of gateway successfully") return ctrl.Result{}, nil } -func (r *gatewayReconciler) deployGateway(gw *gwv1.Gateway, mc configurator.Configurator) (ctrl.Result, error) { +func (r *gatewayReconciler) deployGateway(gw *gwv1.Gateway, mc configurator.Configurator, update *gw.GatewayStatusUpdate) (ctrl.Result, error) { actionConfig := helm.ActionConfig(gw.Namespace, log.Debug().Msgf) + resolveValues := func(object metav1.Object, mc configurator.Configurator) (map[string]interface{}, error) { + gatewayValues, err := r.resolveGatewayValues(object, mc, update) + if err != nil { + return nil, err + } + + parameterValues, err := r.resolveParameterValues(gw, update) + if err != nil { + log.Error().Msgf("Failed to resolve parameter values from ParametersRef: %s, it doesn't take effect", err) + return gatewayValues, nil + } + + if parameterValues == nil { + return gatewayValues, nil + } + + // gateway values take precedence over parameter values, means the values from MeshConfig override the values from ParametersRef + // see the overrides variables for a complete list of values + return chartutil.CoalesceTables(parameterValues, gatewayValues), nil + } + templateClient := helm.TemplateClient( actionConfig, - fmt.Sprintf("fsm-gateway-%s", gw.Namespace), + fmt.Sprintf("fsm-gateway-%s-%s", gw.Namespace, gw.Name), gw.Namespace, r.kubeVersionForTemplate(), ) - if ctrlResult, err := helm.RenderChart(templateClient, gw, chartSource, mc, r.fctx.Client, r.fctx.Scheme, r.resolveValues); err != nil { + if ctrlResult, err := helm.RenderChart(templateClient, gw, chartSource, mc, r.fctx.Client, r.fctx.Scheme, resolveValues); err != nil { defer r.recorder.Eventf(gw, corev1.EventTypeWarning, "Deploy", "Failed to deploy gateway: %s", err) return ctrlResult, err } - defer r.recorder.Eventf(gw, corev1.EventTypeNormal, "Deploy", "Deploy gateway successfully") + + //defer r.recorder.Eventf(gw, corev1.EventTypeNormal, "Deploy", "Deploy gateway successfully") return ctrl.Result{}, nil } @@ -790,7 +912,7 @@ func (r *gatewayReconciler) kubeVersionForTemplate() *chartutil.KubeVersion { return constants.KubeVersion119 } -func (r *gatewayReconciler) resolveValues(object metav1.Object, mc configurator.Configurator) (map[string]interface{}, error) { +func (r *gatewayReconciler) resolveGatewayValues(object metav1.Object, mc configurator.Configurator, update *gw.GatewayStatusUpdate) (map[string]interface{}, error) { gateway, ok := object.(*gwv1.Gateway) if !ok { return nil, fmt.Errorf("object %v is not type of *gwv1.Gateway", object) @@ -798,55 +920,83 @@ func (r *gatewayReconciler) resolveValues(object metav1.Object, mc configurator. log.Debug().Msgf("[GW] Resolving Values ...") - gwBytes, err := ghodssyaml.Marshal(&gatewayValues{ - Gateway: gateway, - Listeners: gwutils.GetValidListenersForGateway(gateway), + // these values are from MeshConfig and Gateway resource, it will not be overridden by values from ParametersRef + gwBytes, err := ghodssyaml.Marshal(map[string]interface{}{ + "fsm": map[string]interface{}{ + "fsmNamespace": mc.GetFSMNamespace(), + "meshName": r.fctx.MeshName, + "gateway": map[string]interface{}{ + "namespace": gateway.Namespace, + "name": gateway.Name, + "listeners": r.listenersForTemplate(gateway, update), + "infrastructure": infraForTemplate(gateway), + "logLevel": mc.GetFSMGatewayLogLevel(), + }, + "image": map[string]interface{}{ + "registry": mc.GetImageRegistry(), + "tag": mc.GetImageTag(), + "pullPolicy": mc.GetImagePullPolicy(), + }, + }, + "hasTCP": hasTCP(gateway), + "hasUDP": hasUDP(gateway), }) if err != nil { - return nil, fmt.Errorf("convert Gateway to yaml, err = %v", err) + return nil, fmt.Errorf("convert values map to yaml, err = %v", err) } + log.Debug().Msgf("\n\nGATEWAY VALUES YAML:\n\n\n%s\n\n", string(gwBytes)) + gwValues, err := chartutil.ReadValues(gwBytes) if err != nil { return nil, err } - gatewayValues := gwValues.AsMap() + return gwValues.AsMap(), nil +} - // these values are from MeshConfig, it will not be overridden by values from ParametersRef - overrides := []string{ - fmt.Sprintf("fsm.image.registry=%s", mc.GetImageRegistry()), - fmt.Sprintf("fsm.image.tag=%s", mc.GetImageTag()), - fmt.Sprintf("fsm.image.pullPolicy=%s", mc.GetImagePullPolicy()), - fmt.Sprintf("fsm.fsmNamespace=%s", mc.GetFSMNamespace()), - fmt.Sprintf("fsm.fsmGateway.logLevel=%s", mc.GetFSMGatewayLogLevel()), - fmt.Sprintf("fsm.meshName=%s", r.fctx.MeshName), - fmt.Sprintf("hasTCP=%t", hasTCP(gateway)), - fmt.Sprintf("hasUDP=%t", hasUDP(gateway)), +func infraForTemplate(gateway *gwv1.Gateway) map[string]map[gwv1.AnnotationKey]gwv1.AnnotationValue { + infra := map[string]map[gwv1.AnnotationKey]gwv1.AnnotationValue{ + "annotations": {}, + "labels": {}, } - for _, ov := range overrides { - if err := strvals.ParseInto(ov, gatewayValues); err != nil { - return nil, err + if gateway.Spec.Infrastructure != nil { + if len(gateway.Spec.Infrastructure.Annotations) > 0 { + infra["annotations"] = gateway.Spec.Infrastructure.Annotations + } + if len(gateway.Spec.Infrastructure.Labels) > 0 { + infra["labels"] = gateway.Spec.Infrastructure.Labels } } - parameterValues, err := r.resolveParameterValues(gateway) - if err != nil { - log.Error().Msgf("Failed to resolve parameter values from ParametersRef: %s, it doesn't take effect", err) - return gatewayValues, nil - } + return infra +} + +func (r *gatewayReconciler) listenersForTemplate(gateway *gwv1.Gateway, update *gw.GatewayStatusUpdate) []listener { + listeners := make([]listener, 0) + for _, l := range gateway.Spec.Listeners { + s := update.GetListenerStatus(string(l.Name)) + + if s == nil { + continue + } + + if !gwutils.IsListenerValid(*s) { + continue + } - if parameterValues == nil { - return gatewayValues, nil + listeners = append(listeners, listener{ + Name: l.Name, + Port: l.Port, + Protocol: l.Protocol, + }) } - // gateway values take precedence over parameter values, means the values from MeshConfig override the values from ParametersRef - // see the overrides variables for a complete list of values - return helmutil.MergeMaps(parameterValues, gatewayValues), nil + return listeners } -func (r *gatewayReconciler) resolveParameterValues(gateway *gwv1.Gateway) (map[string]interface{}, error) { +func (r *gatewayReconciler) resolveParameterValues(gateway *gwv1.Gateway, update *gw.GatewayStatusUpdate) (map[string]interface{}, error) { if gateway.Spec.Infrastructure == nil { return nil, nil } @@ -871,51 +1021,57 @@ func (r *gatewayReconciler) resolveParameterValues(gateway *gwv1.Gateway) (map[s } if err := r.fctx.Get(context.TODO(), key, cm); err != nil { + update.AddCondition( + gwv1.GatewayConditionAccepted, + metav1.ConditionFalse, + gwv1.GatewayReasonInvalidParameters, + fmt.Sprintf("Failed to get ConfigMap %s: %s", key, err), + ) return nil, fmt.Errorf("failed to get Configmap %s: %s", key, err) } if len(cm.Data) == 0 { + update.AddCondition( + gwv1.GatewayConditionAccepted, + metav1.ConditionFalse, + gwv1.GatewayReasonInvalidParameters, + fmt.Sprintf("Configmap %q has no data", key), + ) return nil, fmt.Errorf("configmap %q has no data", key) } valuesYaml, ok := cm.Data["values.yaml"] if !ok { + update.AddCondition( + gwv1.GatewayConditionAccepted, + metav1.ConditionFalse, + gwv1.GatewayReasonInvalidParameters, + fmt.Sprintf("Configmap %q doesn't have required values.yaml", key), + ) return nil, fmt.Errorf("configmap %q has no values.yaml", key) } + log.Debug().Msgf("[GW] values.yaml from ConfigMap %s: \n%s\n", key.String(), valuesYaml) + paramsMap := map[string]interface{}{} if err := yaml.Unmarshal([]byte(valuesYaml), ¶msMap); err != nil { + update.AddCondition( + gwv1.GatewayConditionAccepted, + metav1.ConditionFalse, + gwv1.GatewayReasonInvalidParameters, + fmt.Sprintf("Failed to unmarshal values.yaml of Configmap %s: %s", key, err), + ) return nil, fmt.Errorf("failed to unmarshal values.yaml of Configmap %s: %s", key, err) } - return paramsMap, nil -} - -func (r *gatewayReconciler) setAccepted(gateway *gwv1.Gateway) { - metautil.SetStatusCondition(&gateway.Status.Conditions, metav1.Condition{ - Type: string(gwv1.GatewayConditionAccepted), - Status: metav1.ConditionTrue, - ObservedGeneration: gateway.Generation, - LastTransitionTime: metav1.Time{Time: time.Now()}, - Reason: string(gwv1.GatewayReasonAccepted), - Message: fmt.Sprintf("Gateway %s/%s is accepted.", gateway.Namespace, gateway.Name), - }) -} + log.Debug().Msgf("[GW] values parsed from values.yaml: %v", paramsMap) -func (r *gatewayReconciler) setUnaccepted(gateway *gwv1.Gateway) { - metautil.SetStatusCondition(&gateway.Status.Conditions, metav1.Condition{ - Type: string(gwv1.GatewayConditionAccepted), - Status: metav1.ConditionFalse, - ObservedGeneration: gateway.Generation, - LastTransitionTime: metav1.Time{Time: time.Now()}, - Reason: "Unaccepted", - Message: fmt.Sprintf("Gateway %s/%s is not accepted as it's not the oldest one in namespace %q.", gateway.Namespace, gateway.Name, gateway.Namespace), - }) + return paramsMap, nil } // SetupWithManager sets up the controller with the Manager. func (r *gatewayReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). + if err := ctrl.NewControllerManagedBy(mgr). For(&gwv1.Gateway{}, builder.WithPredicates(predicate.NewPredicateFuncs(func(obj client.Object) bool { gateway, ok := obj.(*gwv1.Gateway) if !ok { @@ -923,11 +1079,9 @@ func (r *gatewayReconciler) SetupWithManager(mgr ctrl.Manager) error { return false } - gatewayClass, err := r.gatewayAPIClient. - GatewayV1(). - GatewayClasses(). - Get(context.TODO(), string(gateway.Spec.GatewayClassName), metav1.GetOptions{}) - if err != nil { + gatewayClass := &gwv1.GatewayClass{} + key := types.NamespacedName{Name: string(gateway.Spec.GatewayClassName)} + if err := r.fctx.Get(context.TODO(), key, gatewayClass); err != nil { log.Error().Msgf("failed to get gatewayclass %s", gateway.Spec.GatewayClassName) return false } @@ -956,7 +1110,15 @@ func (r *gatewayReconciler) SetupWithManager(mgr ctrl.Manager) error { &corev1.ConfigMap{}, handler.EnqueueRequestsFromMapFunc(r.configMapToGateways), ). - Complete(r) + Watches( + &corev1.Secret{}, + handler.EnqueueRequestsFromMapFunc(r.secretToGateways), + ). + Complete(r); err != nil { + return err + } + + return addGatewayIndexers(context.TODO(), mgr) } func (r *gatewayReconciler) gatewayClassToGateways(ctx context.Context, obj client.Object) []reconcile.Request { @@ -966,20 +1128,24 @@ func (r *gatewayReconciler) gatewayClassToGateways(ctx context.Context, obj clie return nil } - if gwutils.IsEffectiveGatewayClass(gatewayClass) { - var gateways gwv1.GatewayList - if err := r.fctx.List(ctx, &gateways); err != nil { + if gwutils.IsAcceptedGatewayClass(gatewayClass) { + c := r.fctx.Manager.GetCache() + gateways := &gwv1.GatewayList{} + if err := c.List(ctx, gateways, &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(constants.ClassGatewayIndex, gatewayClass.Name), + }); err != nil { log.Error().Msgf("error listing gateways: %s", err) return nil } var reconciles []reconcile.Request - for _, gw := range gateways.Items { - if string(gw.Spec.GatewayClassName) == gatewayClass.GetName() { + for _, gwy := range gateways.Items { + gwy := gwy + if gwutils.IsActiveGateway(&gwy) { reconciles = append(reconciles, reconcile.Request{ NamespacedName: types.NamespacedName{ - Namespace: gw.Namespace, - Name: gw.Name, + Namespace: gwy.Namespace, + Name: gwy.Name, }, }) } @@ -998,9 +1164,12 @@ func (r *gatewayReconciler) configMapToGateways(ctx context.Context, object clie return nil } + c := r.fctx.Manager.GetCache() gateways := &gwv1.GatewayList{} - err := r.fctx.List(ctx, gateways, client.InNamespace(cm.Namespace)) - if err != nil { + if err := c.List(ctx, gateways, &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(constants.ConfigMapGatewayIndex, client.ObjectKeyFromObject(cm).String()), + Namespace: cm.Namespace, + }); err != nil { log.Error().Msgf("error listing gateways: %s", err) return nil } @@ -1011,20 +1180,48 @@ func (r *gatewayReconciler) configMapToGateways(ctx context.Context, object clie reconciles := make([]reconcile.Request, 0) for _, gw := range gateways.Items { - if gw.Spec.Infrastructure == nil { - continue + gw := gw + if gwutils.IsActiveGateway(&gw) { + reconciles = append(reconciles, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: gw.Namespace, + Name: gw.Name, + }, + }) } + } - if gw.Spec.Infrastructure.ParametersRef == nil { - continue - } + return reconciles +} + +func (r *gatewayReconciler) secretToGateways(ctx context.Context, object client.Object) []reconcile.Request { + secret, ok := object.(*corev1.Secret) + if !ok { + log.Error().Msgf("unexpected object type: %T", object) + return nil + } + + c := r.fctx.Manager.GetCache() + gateways := &gwv1.GatewayList{} + if err := c.List(ctx, gateways, &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(constants.SecretGatewayIndex, client.ObjectKeyFromObject(secret).String()), + }); err != nil { + log.Error().Msgf("error listing gateways: %s", err) + return nil + } + + if len(gateways.Items) == 0 { + return nil + } - paramRef := gw.Spec.Infrastructure.ParametersRef - if paramRef.Name == cm.Name && paramRef.Group == corev1.GroupName && paramRef.Kind == constants.KubernetesConfigMapKind { + reconciles := make([]reconcile.Request, 0) + for _, gwy := range gateways.Items { + gwy := gwy + if gwutils.IsActiveGateway(&gwy) { reconciles = append(reconciles, reconcile.Request{ NamespacedName: types.NamespacedName{ - Namespace: gw.Namespace, - Name: gw.Name, + Namespace: gwy.Namespace, + Name: gwy.Name, }, }) } @@ -1032,3 +1229,108 @@ func (r *gatewayReconciler) configMapToGateways(ctx context.Context, object clie return reconciles } + +func addGatewayIndexers(ctx context.Context, mgr manager.Manager) error { + if err := mgr.GetFieldIndexer().IndexField(ctx, &gwv1.Gateway{}, constants.SecretGatewayIndex, secretGatewayIndexFunc); err != nil { + return err + } + + if err := mgr.GetFieldIndexer().IndexField(ctx, &gwv1.Gateway{}, constants.ConfigMapGatewayIndex, configMapGatewayIndexFunc); err != nil { + return err + } + + if err := mgr.GetFieldIndexer().IndexField(ctx, &gwv1.Gateway{}, constants.ClassGatewayIndex, func(obj client.Object) []string { + gateway := obj.(*gwv1.Gateway) + return []string{string(gateway.Spec.GatewayClassName)} + }); err != nil { + return err + } + + return nil +} + +func secretGatewayIndexFunc(obj client.Object) []string { + gateway := obj.(*gwv1.Gateway) + var secretReferences []string + for _, listener := range gateway.Spec.Listeners { + if listener.Protocol != gwv1.TLSProtocolType && listener.Protocol != gwv1.HTTPSProtocolType { + continue + } + + if listener.TLS == nil || *listener.TLS.Mode != gwv1.TLSModeTerminate { + continue + } + + for _, cert := range listener.TLS.CertificateRefs { + if *cert.Kind == constants.KubernetesSecretKind { + secretReferences = append(secretReferences, + types.NamespacedName{ + Namespace: gwutils.NamespaceDerefOr(cert.Namespace, gateway.Namespace), + Name: string(cert.Name), + }.String(), + ) + } + } + + if listener.TLS.FrontendValidation != nil { + for _, ca := range listener.TLS.FrontendValidation.CACertificateRefs { + if ca.Kind == constants.KubernetesSecretKind { + secretReferences = append(secretReferences, + types.NamespacedName{ + Namespace: gwutils.NamespaceDerefOr(ca.Namespace, gateway.Namespace), + Name: string(ca.Name), + }.String(), + ) + } + } + } + } + + return secretReferences +} + +func configMapGatewayIndexFunc(obj client.Object) []string { + gateway := obj.(*gwv1.Gateway) + var cmRefs []string + + // check against listeners + for _, listener := range gateway.Spec.Listeners { + if listener.Protocol != gwv1.TLSProtocolType && listener.Protocol != gwv1.HTTPSProtocolType { + continue + } + + if listener.TLS == nil || *listener.TLS.Mode != gwv1.TLSModeTerminate { + continue + } + + if listener.TLS.FrontendValidation == nil { + continue + } + + for _, ca := range listener.TLS.FrontendValidation.CACertificateRefs { + if ca.Kind == constants.KubernetesConfigMapKind { + cmRefs = append(cmRefs, + types.NamespacedName{ + Namespace: gwutils.NamespaceDerefOr(ca.Namespace, gateway.Namespace), + Name: string(ca.Name), + }.String(), + ) + } + } + } + + // check against infrastructure ParametersRef + if gateway.Spec.Infrastructure != nil && gateway.Spec.Infrastructure.ParametersRef != nil { + parametersRef := gateway.Spec.Infrastructure.ParametersRef + if parametersRef.Kind == constants.KubernetesConfigMapKind { + cmRefs = append(cmRefs, + types.NamespacedName{ + Namespace: gateway.Namespace, + Name: string(parametersRef.Name), + }.String(), + ) + } + } + + return cmRefs +} diff --git a/pkg/controllers/gateway/v1/gatewayclass_controller.go b/pkg/controllers/gateway/v1/gatewayclass_controller.go index 21a09eef9..e134d064e 100644 --- a/pkg/controllers/gateway/v1/gatewayclass_controller.go +++ b/pkg/controllers/gateway/v1/gatewayclass_controller.go @@ -27,12 +27,11 @@ package v1 import ( "context" "fmt" - "sort" "time" - "github.com/flomesh-io/fsm/pkg/gateway/status" + "sigs.k8s.io/controller-runtime/pkg/manager" - gwclient "sigs.k8s.io/gateway-api/pkg/client/clientset/versioned" + "github.com/flomesh-io/fsm/pkg/gateway/status" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -45,19 +44,14 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" gwv1 "sigs.k8s.io/gateway-api/apis/v1" - "github.com/flomesh-io/fsm/pkg/apis/gateway" "github.com/flomesh-io/fsm/pkg/constants" fctx "github.com/flomesh-io/fsm/pkg/context" "github.com/flomesh-io/fsm/pkg/controllers" - "github.com/flomesh-io/fsm/pkg/gateway/utils" - - gatewayApiClientset "sigs.k8s.io/gateway-api/pkg/client/clientset/versioned" ) type gatewayClassReconciler struct { - recorder record.EventRecorder - fctx *fctx.ControllerContext - gatewayAPIClient gwclient.Interface + recorder record.EventRecorder + fctx *fctx.ControllerContext } func (r *gatewayClassReconciler) NeedLeaderElection() bool { @@ -67,9 +61,8 @@ func (r *gatewayClassReconciler) NeedLeaderElection() bool { // NewGatewayClassReconciler returns a new reconciler for GatewayClass func NewGatewayClassReconciler(ctx *fctx.ControllerContext) controllers.Reconciler { return &gatewayClassReconciler{ - recorder: ctx.Manager.GetEventRecorderFor("GatewayClass"), - fctx: ctx, - gatewayAPIClient: gatewayApiClientset.NewForConfigOrDie(ctx.KubeConfig), + recorder: ctx.Manager.GetEventRecorderFor("GatewayClass"), + fctx: ctx, } } @@ -104,169 +97,32 @@ func (r *gatewayClassReconciler) Reconcile(ctx context.Context, req ctrl.Request return ctrl.Result{}, nil } - // Accept all GatewayClasses those ControllerName is flomesh.io/gateway-controller - //r.setAcceptedStatus(gatewayClass) - //result, err := r.updateStatus(ctx, gatewayClass, gwv1.GatewayClassConditionStatusAccepted) - //if err != nil { - // return result, err - //} + // ignore if the GatewayClass is not managed by the FSM GatewayController + if gatewayClass.Spec.ControllerName != constants.GatewayController { + return ctrl.Result{}, nil + } r.fctx.StatusUpdater.Send(status.Update{ - Resource: gatewayClass, + Resource: &gwv1.GatewayClass{}, NamespacedName: client.ObjectKeyFromObject(gatewayClass), Mutator: status.MutatorFunc(func(obj client.Object) client.Object { class, ok := obj.(*gwv1.GatewayClass) if !ok { - log.Error().Msgf("unsupported object type %T", obj) + log.Error().Msgf("Unexpected object type %T", obj) } classCopy := class.DeepCopy() - r.setAcceptedStatus(classCopy) + r.setAccepted(classCopy) return classCopy }), }) - gatewayClassList, err := r.gatewayAPIClient.GatewayV1(). - GatewayClasses(). - List(ctx, metav1.ListOptions{}) - if err != nil { - log.Error().Msgf("failed list gatewayclasses: %s", err) - return ctrl.Result{}, err - } - - // If there's multiple GatewayClasses whose ControllerName is flomesh.io/gateway-controller, the oldest is set to active and the rest are set to inactive - r.updateActiveStatus(gatewayClassList) - //for _, class := range r.updateActiveStatus(gatewayClassList) { - //result, err := r.updateStatus(ctx, class, gateway.GatewayClassConditionStatusActive) - //if err != nil { - // return result, err - //} - //} - - // As status of all GatewayClasses have been updated, just send the event - r.fctx.GatewayEventHandler.OnAdd(&gwv1.GatewayClass{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: req.Namespace, - Name: req.Name, - }}, - false, - ) - return ctrl.Result{}, nil } -//func (r *gatewayClassReconciler) updateStatus(ctx context.Context, class *gwv1.GatewayClass, status gwv1.GatewayClassConditionType) (ctrl.Result, error) { -// if err := r.fctx.Status().Update(ctx, class); err != nil { -// //defer r.recorder.Eventf(class, corev1.EventTypeWarning, "UpdateStatus", "Failed to update status of GatewayClass: %s", err) -// return ctrl.Result{}, err -// } -// -// switch status { -// case gwv1.GatewayClassConditionStatusAccepted: -// if utils.IsAcceptedGatewayClass(class) { -// defer r.recorder.Eventf(class, corev1.EventTypeNormal, "Accepted", "GatewayClass is accepted") -// } else { -// defer r.recorder.Eventf(class, corev1.EventTypeNormal, "Rejected", "GatewayClass is rejected") -// } -// case gateway.GatewayClassConditionStatusActive: -// if utils.IsActiveGatewayClass(class) { -// defer r.recorder.Eventf(class, corev1.EventTypeNormal, "Active", "GatewayClass is set to active") -// } else { -// defer r.recorder.Eventf(class, corev1.EventTypeNormal, "Inactive", "GatewayClass is set to inactive") -// } -// } -// -// return ctrl.Result{}, nil -//} - -func (r *gatewayClassReconciler) setAcceptedStatus(gatewayClass *gwv1.GatewayClass) { - if gatewayClass.Spec.ControllerName == constants.GatewayController { - r.setAccepted(gatewayClass) - } - // Ignore other GatewayClass whose ControllerName is not flomesh.io/gateway-controller - // as they are not supported and should not be processed by fsm - //else { - // r.setRejected(gatewayClass) - //} -} - -func (r *gatewayClassReconciler) updateActiveStatus(list *gwv1.GatewayClassList) { - acceptedClasses := make([]*gwv1.GatewayClass, 0) - for _, class := range list.Items { - class := class // fix lint GO-LOOP-REF - if class.Spec.ControllerName == constants.GatewayController && utils.IsAcceptedGatewayClass(&class) { - acceptedClasses = append(acceptedClasses, &class) - } - } - - sort.Slice(acceptedClasses, func(i, j int) bool { - if acceptedClasses[i].CreationTimestamp.Time.Equal(acceptedClasses[j].CreationTimestamp.Time) { - return acceptedClasses[i].Name < acceptedClasses[j].Name - } - - return acceptedClasses[i].CreationTimestamp.Time.Before(acceptedClasses[j].CreationTimestamp.Time) - }) - - //statusChangedClasses := make([]*gwv1.GatewayClass, 0) - for i, class := range acceptedClasses { - // ONLY the oldest GatewayClass is active - if i == 0 { - if !utils.IsActiveGatewayClass(class) { - //r.setActive(acceptedClasses[i]) - //statusChangedClasses = append(statusChangedClasses, acceptedClasses[i]) - r.fctx.StatusUpdater.Send(status.Update{ - Resource: class, - NamespacedName: client.ObjectKeyFromObject(class), - Mutator: status.MutatorFunc(func(obj client.Object) client.Object { - clazz, ok := obj.(*gwv1.GatewayClass) - if !ok { - log.Error().Msgf("unsupported object type %T", obj) - } - classCopy := clazz.DeepCopy() - r.setActive(classCopy) - - return classCopy - }), - }) - } - continue - } - - if utils.IsActiveGatewayClass(class) { - //r.setInactive(acceptedClasses[i]) - //statusChangedClasses = append(statusChangedClasses, acceptedClasses[i]) - r.fctx.StatusUpdater.Send(status.Update{ - Resource: class, - NamespacedName: client.ObjectKeyFromObject(class), - Mutator: status.MutatorFunc(func(obj client.Object) client.Object { - clazz, ok := obj.(*gwv1.GatewayClass) - if !ok { - log.Error().Msgf("unsupported object type %T", obj) - } - classCopy := clazz.DeepCopy() - r.setInactive(classCopy) - - return classCopy - }), - }) - } - } - - //return statusChangedClasses -} - -//func (r *gatewayClassReconciler) setRejected(gatewayClass *gwv1.GatewayClass) { -// metautil.SetStatusCondition(&gatewayClass.Status.Conditions, metav1.Condition{ -// Type: string(gwv1.GatewayClassConditionStatusAccepted), -// Status: metav1.ConditionFalse, -// ObservedGeneration: gatewayClass.Generation, -// LastTransitionTime: metav1.Time{Time: time.Now()}, -// Reason: "Rejected", -// Message: fmt.Sprintf("GatewayClass %q is rejected as ControllerName %q is not supported.", gatewayClass.Name, gatewayClass.Spec.ControllerName), -// }) -//} - func (r *gatewayClassReconciler) setAccepted(gatewayClass *gwv1.GatewayClass) { + defer r.recorder.Eventf(gatewayClass, corev1.EventTypeNormal, "Accepted", "GatewayClass is accepted") + metautil.SetStatusCondition(&gatewayClass.Status.Conditions, metav1.Condition{ Type: string(gwv1.GatewayClassConditionStatusAccepted), Status: metav1.ConditionTrue, @@ -275,30 +131,6 @@ func (r *gatewayClassReconciler) setAccepted(gatewayClass *gwv1.GatewayClass) { Reason: string(gwv1.GatewayClassReasonAccepted), Message: fmt.Sprintf("GatewayClass %q is accepted.", gatewayClass.Name), }) - defer r.recorder.Eventf(gatewayClass, corev1.EventTypeNormal, "Accepted", "GatewayClass is accepted") -} - -func (r *gatewayClassReconciler) setActive(gatewayClass *gwv1.GatewayClass) { - metautil.SetStatusCondition(&gatewayClass.Status.Conditions, metav1.Condition{ - Type: string(gateway.GatewayClassConditionStatusActive), - Status: metav1.ConditionTrue, - ObservedGeneration: gatewayClass.Generation, - LastTransitionTime: metav1.Time{Time: time.Now()}, - Reason: string(gateway.GatewayClassReasonActive), - Message: fmt.Sprintf("GatewayClass %q is set to active.", gatewayClass.Name), - }) - defer r.recorder.Eventf(gatewayClass, corev1.EventTypeNormal, "Active", "GatewayClass is set to active") -} -func (r *gatewayClassReconciler) setInactive(gatewayClass *gwv1.GatewayClass) { - metautil.SetStatusCondition(&gatewayClass.Status.Conditions, metav1.Condition{ - Type: string(gateway.GatewayClassConditionStatusActive), - Status: metav1.ConditionFalse, - ObservedGeneration: gatewayClass.Generation, - LastTransitionTime: metav1.Time{Time: time.Now()}, - Reason: string(gateway.GatewayClassReasonInactive), - Message: fmt.Sprintf("GatewayClass %q is inactive as there's already an active GatewayClass.", gatewayClass.Name), - }) - defer r.recorder.Eventf(gatewayClass, corev1.EventTypeNormal, "Inactive", "GatewayClass is set to inactive") } // SetupWithManager sets up the controller with the Manager. @@ -313,7 +145,22 @@ func (r *gatewayClassReconciler) SetupWithManager(mgr ctrl.Manager) error { return gatewayClass.Spec.ControllerName == constants.GatewayController }) - return ctrl.NewControllerManagedBy(mgr). + if err := ctrl.NewControllerManagedBy(mgr). For(&gwv1.GatewayClass{}, builder.WithPredicates(gwclsPrct)). - Complete(r) + Complete(r); err != nil { + return err + } + + return addGatewayClassIndexers(context.Background(), mgr) +} + +func addGatewayClassIndexers(ctx context.Context, mgr manager.Manager) error { + if err := mgr.GetFieldIndexer().IndexField(ctx, &gwv1.GatewayClass{}, constants.ControllerGatewayClassIndex, func(obj client.Object) []string { + cls := obj.(*gwv1.GatewayClass) + return []string{string(cls.Spec.ControllerName)} + }); err != nil { + return err + } + + return nil } diff --git a/pkg/controllers/gateway/v1/grpcroute_controller.go b/pkg/controllers/gateway/v1/grpcroute_controller.go index 8bae7f172..99b3ed4e4 100644 --- a/pkg/controllers/gateway/v1/grpcroute_controller.go +++ b/pkg/controllers/gateway/v1/grpcroute_controller.go @@ -27,6 +27,17 @@ package v1 import ( "context" + "github.com/flomesh-io/fsm/pkg/gateway/status/route" + + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + gwutils "github.com/flomesh-io/fsm/pkg/gateway/utils" + + "sigs.k8s.io/controller-runtime/pkg/manager" + + "github.com/flomesh-io/fsm/pkg/constants" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -37,13 +48,12 @@ import ( fctx "github.com/flomesh-io/fsm/pkg/context" "github.com/flomesh-io/fsm/pkg/controllers" - "github.com/flomesh-io/fsm/pkg/gateway/status" ) type grpcRouteReconciler struct { recorder record.EventRecorder fctx *fctx.ControllerContext - statusProcessor *status.RouteStatusProcessor + statusProcessor *route.RouteStatusProcessor } func (r *grpcRouteReconciler) NeedLeaderElection() bool { @@ -55,7 +65,7 @@ func NewGRPCRouteReconciler(ctx *fctx.ControllerContext) controllers.Reconciler return &grpcRouteReconciler{ recorder: ctx.Manager.GetEventRecorderFor("GRPCRoute"), fctx: ctx, - statusProcessor: &status.RouteStatusProcessor{Informers: ctx.InformerCollection}, + statusProcessor: route.NewRouteStatusProcessor(ctx.Manager.GetCache()), } } @@ -77,18 +87,17 @@ func (r *grpcRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( return ctrl.Result{}, nil } - routeStatus, err := r.statusProcessor.ProcessRouteStatus(ctx, grpcRoute) - if err != nil { + rsu := route.NewRouteStatusUpdate( + grpcRoute, + &grpcRoute.ObjectMeta, + &grpcRoute.TypeMeta, + grpcRoute.Spec.Hostnames, + gwutils.ToSlicePtr(grpcRoute.Status.Parents), + ) + if err := r.statusProcessor.Process(ctx, r.fctx.StatusUpdater, rsu, grpcRoute.Spec.ParentRefs); err != nil { return ctrl.Result{}, err } - if len(routeStatus) > 0 { - grpcRoute.Status.Parents = routeStatus - if err := r.fctx.Status().Update(ctx, grpcRoute); err != nil { - return ctrl.Result{}, err - } - } - r.fctx.GatewayEventHandler.OnAdd(grpcRoute, false) return ctrl.Result{}, nil @@ -96,7 +105,88 @@ func (r *grpcRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( // SetupWithManager sets up the controller with the Manager. func (r *grpcRouteReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). + if err := ctrl.NewControllerManagedBy(mgr). For(&gwv1.GRPCRoute{}). - Complete(r) + Complete(r); err != nil { + return err + } + + return addGRPCRouteIndexers(context.Background(), mgr) +} + +func addGRPCRouteIndexers(ctx context.Context, mgr manager.Manager) error { + if err := mgr.GetFieldIndexer().IndexField(ctx, &gwv1.GRPCRoute{}, constants.GatewayGRPCRouteIndex, gatewayGRPCRouteIndexFunc); err != nil { + return err + } + + if err := mgr.GetFieldIndexer().IndexField(ctx, &gwv1.GRPCRoute{}, constants.BackendGRPCRouteIndex, backendGRPCRouteIndexFunc); err != nil { + return err + } + + return nil +} + +func gatewayGRPCRouteIndexFunc(obj client.Object) []string { + grpcroute := obj.(*gwv1.GRPCRoute) + var gateways []string + for _, parent := range grpcroute.Spec.ParentRefs { + if parent.Kind == nil || string(*parent.Kind) == constants.GatewayAPIGatewayKind { + // If an explicit Gateway namespace is not provided, use the GRPCRoute namespace to + // lookup the provided Gateway Name. + gateways = append(gateways, + types.NamespacedName{ + Namespace: gwutils.NamespaceDerefOr(parent.Namespace, grpcroute.Namespace), + Name: string(parent.Name), + }.String(), + ) + } + } + return gateways +} + +func backendGRPCRouteIndexFunc(obj client.Object) []string { + grpcroute := obj.(*gwv1.GRPCRoute) + var backendRefs []string + for _, rule := range grpcroute.Spec.Rules { + for _, backend := range rule.BackendRefs { + if backend.Kind == nil || string(*backend.Kind) == constants.KubernetesServiceKind { + backendRefs = append(backendRefs, + types.NamespacedName{ + Namespace: gwutils.NamespaceDerefOr(backend.Namespace, grpcroute.Namespace), + Name: string(backend.Name), + }.String(), + ) + } + + for _, filter := range backend.Filters { + if filter.Type == gwv1.GRPCRouteFilterRequestMirror { + if filter.RequestMirror.BackendRef.Kind == nil || string(*filter.RequestMirror.BackendRef.Kind) == constants.KubernetesServiceKind { + mirror := filter.RequestMirror.BackendRef + backendRefs = append(backendRefs, + types.NamespacedName{ + Namespace: gwutils.NamespaceDerefOr(mirror.Namespace, grpcroute.Namespace), + Name: string(mirror.Name), + }.String(), + ) + } + } + } + } + + for _, filter := range rule.Filters { + if filter.Type == gwv1.GRPCRouteFilterRequestMirror { + if filter.RequestMirror.BackendRef.Kind == nil || string(*filter.RequestMirror.BackendRef.Kind) == constants.KubernetesServiceKind { + mirror := filter.RequestMirror.BackendRef + backendRefs = append(backendRefs, + types.NamespacedName{ + Namespace: gwutils.NamespaceDerefOr(mirror.Namespace, grpcroute.Namespace), + Name: string(mirror.Name), + }.String(), + ) + } + } + } + } + + return backendRefs } diff --git a/pkg/controllers/gateway/v1/httproute_controller.go b/pkg/controllers/gateway/v1/httproute_controller.go index 045723a68..4e1e3eef0 100644 --- a/pkg/controllers/gateway/v1/httproute_controller.go +++ b/pkg/controllers/gateway/v1/httproute_controller.go @@ -27,6 +27,15 @@ package v1 import ( "context" + "github.com/flomesh-io/fsm/pkg/gateway/status/route" + + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" + + "github.com/flomesh-io/fsm/pkg/constants" + gwutils "github.com/flomesh-io/fsm/pkg/gateway/utils" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/record" @@ -36,13 +45,12 @@ import ( fctx "github.com/flomesh-io/fsm/pkg/context" "github.com/flomesh-io/fsm/pkg/controllers" - "github.com/flomesh-io/fsm/pkg/gateway/status" ) type httpRouteReconciler struct { recorder record.EventRecorder fctx *fctx.ControllerContext - statusProcessor *status.RouteStatusProcessor + statusProcessor *route.RouteStatusProcessor } func (r *httpRouteReconciler) NeedLeaderElection() bool { @@ -54,7 +62,7 @@ func NewHTTPRouteReconciler(ctx *fctx.ControllerContext) controllers.Reconciler return &httpRouteReconciler{ recorder: ctx.Manager.GetEventRecorderFor("HTTPRoute"), fctx: ctx, - statusProcessor: &status.RouteStatusProcessor{Informers: ctx.InformerCollection}, + statusProcessor: route.NewRouteStatusProcessor(ctx.Manager.GetCache()), } } @@ -76,18 +84,17 @@ func (r *httpRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( return ctrl.Result{}, nil } - routeStatus, err := r.statusProcessor.ProcessRouteStatus(ctx, httpRoute) - if err != nil { + rsu := route.NewRouteStatusUpdate( + httpRoute, + &httpRoute.ObjectMeta, + &httpRoute.TypeMeta, + httpRoute.Spec.Hostnames, + gwutils.ToSlicePtr(httpRoute.Status.Parents), + ) + if err := r.statusProcessor.Process(ctx, r.fctx.StatusUpdater, rsu, httpRoute.Spec.ParentRefs); err != nil { return ctrl.Result{}, err } - if len(routeStatus) > 0 { - httpRoute.Status.Parents = routeStatus - if err := r.fctx.Status().Update(ctx, httpRoute); err != nil { - return ctrl.Result{}, err - } - } - r.fctx.GatewayEventHandler.OnAdd(httpRoute, false) return ctrl.Result{}, nil @@ -95,7 +102,87 @@ func (r *httpRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( // SetupWithManager sets up the controller with the Manager. func (r *httpRouteReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). + if err := ctrl.NewControllerManagedBy(mgr). For(&gwv1.HTTPRoute{}). - Complete(r) + Complete(r); err != nil { + return err + } + + return addHTTPRouteIndexers(context.Background(), mgr) +} + +func addHTTPRouteIndexers(ctx context.Context, mgr manager.Manager) error { + if err := mgr.GetFieldIndexer().IndexField(ctx, &gwv1.HTTPRoute{}, constants.GatewayHTTPRouteIndex, gatewayHTTPRouteIndexFunc); err != nil { + return err + } + + if err := mgr.GetFieldIndexer().IndexField(ctx, &gwv1.HTTPRoute{}, constants.BackendHTTPRouteIndex, backendHTTPRouteIndexFunc); err != nil { + return err + } + + return nil +} + +func gatewayHTTPRouteIndexFunc(obj client.Object) []string { + httproute := obj.(*gwv1.HTTPRoute) + var gateways []string + for _, parent := range httproute.Spec.ParentRefs { + if parent.Kind == nil || string(*parent.Kind) == constants.GatewayAPIGatewayKind { + gateways = append(gateways, + types.NamespacedName{ + Namespace: gwutils.NamespaceDerefOr(parent.Namespace, httproute.Namespace), + Name: string(parent.Name), + }.String(), + ) + } + } + + return gateways +} + +func backendHTTPRouteIndexFunc(obj client.Object) []string { + httproute := obj.(*gwv1.HTTPRoute) + var backendRefs []string + for _, rule := range httproute.Spec.Rules { + for _, backend := range rule.BackendRefs { + if backend.Kind == nil || string(*backend.Kind) == constants.KubernetesServiceKind { + backendRefs = append(backendRefs, + types.NamespacedName{ + Namespace: gwutils.NamespaceDerefOr(backend.Namespace, httproute.Namespace), + Name: string(backend.Name), + }.String(), + ) + } + + for _, filter := range backend.Filters { + if filter.Type == gwv1.HTTPRouteFilterRequestMirror { + if filter.RequestMirror.BackendRef.Kind == nil || string(*filter.RequestMirror.BackendRef.Kind) == constants.KubernetesServiceKind { + mirror := filter.RequestMirror.BackendRef + backendRefs = append(backendRefs, + types.NamespacedName{ + Namespace: gwutils.NamespaceDerefOr(mirror.Namespace, httproute.Namespace), + Name: string(mirror.Name), + }.String(), + ) + } + } + } + } + + for _, filter := range rule.Filters { + if filter.Type == gwv1.HTTPRouteFilterRequestMirror { + if filter.RequestMirror.BackendRef.Kind == nil || string(*filter.RequestMirror.BackendRef.Kind) == constants.KubernetesServiceKind { + mirror := filter.RequestMirror.BackendRef + backendRefs = append(backendRefs, + types.NamespacedName{ + Namespace: gwutils.NamespaceDerefOr(mirror.Namespace, httproute.Namespace), + Name: string(mirror.Name), + }.String(), + ) + } + } + } + } + + return backendRefs } diff --git a/pkg/controllers/gateway/v1/types.go b/pkg/controllers/gateway/v1/types.go index 8ed7cf8aa..dda276291 100644 --- a/pkg/controllers/gateway/v1/types.go +++ b/pkg/controllers/gateway/v1/types.go @@ -25,8 +25,13 @@ // Package v1 contains controller logic for the Gateway API v1. package v1 -import "github.com/flomesh-io/fsm/pkg/logger" +import ( + "github.com/flomesh-io/fsm/pkg/logger" + "github.com/flomesh-io/fsm/pkg/webhook" +) var ( log = logger.New("gatewayapi-controller/v1") ) + +var isValidHostname = webhook.IsValidHostname diff --git a/pkg/controllers/gateway/v1alpha2/tcproute_controller.go b/pkg/controllers/gateway/v1alpha2/tcproute_controller.go index 293035e6e..7460432e5 100644 --- a/pkg/controllers/gateway/v1alpha2/tcproute_controller.go +++ b/pkg/controllers/gateway/v1alpha2/tcproute_controller.go @@ -27,6 +27,15 @@ package v1alpha2 import ( "context" + "github.com/flomesh-io/fsm/pkg/gateway/status/route" + + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" + + "github.com/flomesh-io/fsm/pkg/constants" + gwutils "github.com/flomesh-io/fsm/pkg/gateway/utils" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/record" @@ -36,13 +45,12 @@ import ( fctx "github.com/flomesh-io/fsm/pkg/context" "github.com/flomesh-io/fsm/pkg/controllers" - "github.com/flomesh-io/fsm/pkg/gateway/status" ) type tcpRouteReconciler struct { recorder record.EventRecorder fctx *fctx.ControllerContext - statusProcessor *status.RouteStatusProcessor + statusProcessor *route.RouteStatusProcessor } func (r *tcpRouteReconciler) NeedLeaderElection() bool { @@ -54,7 +62,7 @@ func NewTCPRouteReconciler(ctx *fctx.ControllerContext) controllers.Reconciler { return &tcpRouteReconciler{ recorder: ctx.Manager.GetEventRecorderFor("TCPRoute"), fctx: ctx, - statusProcessor: &status.RouteStatusProcessor{Informers: ctx.InformerCollection}, + statusProcessor: route.NewRouteStatusProcessor(ctx.Manager.GetCache()), } } @@ -76,18 +84,17 @@ func (r *tcpRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c return ctrl.Result{}, nil } - routeStatus, err := r.statusProcessor.ProcessRouteStatus(ctx, tcpRoute) - if err != nil { + rsu := route.NewRouteStatusUpdate( + tcpRoute, + &tcpRoute.ObjectMeta, + &tcpRoute.TypeMeta, + nil, + gwutils.ToSlicePtr(tcpRoute.Status.Parents), + ) + if err := r.statusProcessor.Process(ctx, r.fctx.StatusUpdater, rsu, tcpRoute.Spec.ParentRefs); err != nil { return ctrl.Result{}, err } - if len(routeStatus) > 0 { - tcpRoute.Status.Parents = routeStatus - if err := r.fctx.Status().Update(ctx, tcpRoute); err != nil { - return ctrl.Result{}, err - } - } - r.fctx.GatewayEventHandler.OnAdd(tcpRoute, false) return ctrl.Result{}, nil @@ -95,7 +102,55 @@ func (r *tcpRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c // SetupWithManager sets up the controller with the Manager. func (r *tcpRouteReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). + if err := ctrl.NewControllerManagedBy(mgr). For(&gwv1alpha2.TCPRoute{}). - Complete(r) + Complete(r); err != nil { + return err + } + + return addTCPRouteIndexers(context.Background(), mgr) +} + +func addTCPRouteIndexers(ctx context.Context, mgr manager.Manager) error { + if err := mgr.GetFieldIndexer().IndexField(ctx, &gwv1alpha2.TCPRoute{}, constants.GatewayTCPRouteIndex, func(obj client.Object) []string { + tcpRoute := obj.(*gwv1alpha2.TCPRoute) + var gateways []string + for _, parent := range tcpRoute.Spec.ParentRefs { + if string(*parent.Kind) == constants.GatewayAPIGatewayKind { + gateways = append(gateways, + types.NamespacedName{ + Namespace: gwutils.NamespaceDerefOr(parent.Namespace, tcpRoute.Namespace), + Name: string(parent.Name), + }.String(), + ) + } + } + return gateways + }); err != nil { + return err + } + + if err := mgr.GetFieldIndexer().IndexField(ctx, &gwv1alpha2.TCPRoute{}, constants.BackendTCPRouteIndex, backendTCPRouteIndexFunc); err != nil { + return err + } + return nil +} + +func backendTCPRouteIndexFunc(obj client.Object) []string { + tcpRoute := obj.(*gwv1alpha2.TCPRoute) + var backendRefs []string + for _, rule := range tcpRoute.Spec.Rules { + for _, backend := range rule.BackendRefs { + if backend.Kind == nil || string(*backend.Kind) == constants.KubernetesServiceKind { + backendRefs = append(backendRefs, + types.NamespacedName{ + Namespace: gwutils.NamespaceDerefOr(backend.Namespace, tcpRoute.Namespace), + Name: string(backend.Name), + }.String(), + ) + } + } + } + + return backendRefs } diff --git a/pkg/controllers/gateway/v1alpha2/tlsroute_controller.go b/pkg/controllers/gateway/v1alpha2/tlsroute_controller.go index ff1465399..012774aaf 100644 --- a/pkg/controllers/gateway/v1alpha2/tlsroute_controller.go +++ b/pkg/controllers/gateway/v1alpha2/tlsroute_controller.go @@ -27,6 +27,17 @@ package v1alpha2 import ( "context" + "github.com/flomesh-io/fsm/pkg/gateway/status/route" + + gwutils "github.com/flomesh-io/fsm/pkg/gateway/utils" + + "sigs.k8s.io/controller-runtime/pkg/manager" + + "github.com/flomesh-io/fsm/pkg/constants" + + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/record" @@ -36,13 +47,12 @@ import ( fctx "github.com/flomesh-io/fsm/pkg/context" "github.com/flomesh-io/fsm/pkg/controllers" - "github.com/flomesh-io/fsm/pkg/gateway/status" ) type tlsRouteReconciler struct { recorder record.EventRecorder fctx *fctx.ControllerContext - statusProcessor *status.RouteStatusProcessor + statusProcessor *route.RouteStatusProcessor } func (r *tlsRouteReconciler) NeedLeaderElection() bool { @@ -54,7 +64,7 @@ func NewTLSRouteReconciler(ctx *fctx.ControllerContext) controllers.Reconciler { return &tlsRouteReconciler{ recorder: ctx.Manager.GetEventRecorderFor("TLSRoute"), fctx: ctx, - statusProcessor: &status.RouteStatusProcessor{Informers: ctx.InformerCollection}, + statusProcessor: route.NewRouteStatusProcessor(ctx.Manager.GetCache()), } } @@ -76,18 +86,17 @@ func (r *tlsRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c return ctrl.Result{}, nil } - routeStatus, err := r.statusProcessor.ProcessRouteStatus(ctx, tlsRoute) - if err != nil { + rsu := route.NewRouteStatusUpdate( + tlsRoute, + &tlsRoute.ObjectMeta, + &tlsRoute.TypeMeta, + tlsRoute.Spec.Hostnames, + gwutils.ToSlicePtr(tlsRoute.Status.Parents), + ) + if err := r.statusProcessor.Process(ctx, r.fctx.StatusUpdater, rsu, tlsRoute.Spec.ParentRefs); err != nil { return ctrl.Result{}, err } - if len(routeStatus) > 0 { - tlsRoute.Status.Parents = routeStatus - if err := r.fctx.Status().Update(ctx, tlsRoute); err != nil { - return ctrl.Result{}, err - } - } - r.fctx.GatewayEventHandler.OnAdd(tlsRoute, false) return ctrl.Result{}, nil @@ -95,7 +104,55 @@ func (r *tlsRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c // SetupWithManager sets up the controller with the Manager. func (r *tlsRouteReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). + if err := ctrl.NewControllerManagedBy(mgr). For(&gwv1alpha2.TLSRoute{}). - Complete(r) + Complete(r); err != nil { + return err + } + + return addTLSRouteIndexers(context.Background(), mgr) +} + +func addTLSRouteIndexers(ctx context.Context, mgr manager.Manager) error { + if err := mgr.GetFieldIndexer().IndexField(ctx, &gwv1alpha2.TLSRoute{}, constants.GatewayTLSRouteIndex, func(obj client.Object) []string { + tlsRoute := obj.(*gwv1alpha2.TLSRoute) + var gateways []string + for _, parent := range tlsRoute.Spec.ParentRefs { + if string(*parent.Kind) == constants.GatewayAPIGatewayKind { + gateways = append(gateways, + types.NamespacedName{ + Namespace: gwutils.NamespaceDerefOr(parent.Namespace, tlsRoute.Namespace), + Name: string(parent.Name), + }.String(), + ) + } + } + return gateways + }); err != nil { + return err + } + + if err := mgr.GetFieldIndexer().IndexField(ctx, &gwv1alpha2.TLSRoute{}, constants.BackendTLSRouteIndex, backendTLSRouteIndexFunc); err != nil { + return err + } + return nil +} + +func backendTLSRouteIndexFunc(obj client.Object) []string { + tlsroute := obj.(*gwv1alpha2.TLSRoute) + var backendRefs []string + for _, rule := range tlsroute.Spec.Rules { + for _, backend := range rule.BackendRefs { + if backend.Kind == nil || string(*backend.Kind) == constants.KubernetesServiceKind { + backendRefs = append(backendRefs, + types.NamespacedName{ + Namespace: gwutils.NamespaceDerefOr(backend.Namespace, tlsroute.Namespace), + Name: string(backend.Name), + }.String(), + ) + } + } + } + + return backendRefs } diff --git a/pkg/controllers/gateway/v1alpha2/types.go b/pkg/controllers/gateway/v1alpha2/types.go index 7fa442c5f..8c1590ab8 100644 --- a/pkg/controllers/gateway/v1alpha2/types.go +++ b/pkg/controllers/gateway/v1alpha2/types.go @@ -1,8 +1,6 @@ // Package v1alpha2 contains controllers logic for the Gateway API v1alpha2. package v1alpha2 -//import "github.com/flomesh-io/fsm/pkg/logger" -// //var ( -// log = logger.New("gatewayapi-v1alpha2-controller") +// log = logger.New("gatewayapi-controller/v1alpha2") //) diff --git a/pkg/controllers/gateway/v1alpha2/udproute_controller.go b/pkg/controllers/gateway/v1alpha2/udproute_controller.go index 26167038c..6c11a0d1e 100644 --- a/pkg/controllers/gateway/v1alpha2/udproute_controller.go +++ b/pkg/controllers/gateway/v1alpha2/udproute_controller.go @@ -27,6 +27,15 @@ package v1alpha2 import ( "context" + "github.com/flomesh-io/fsm/pkg/gateway/status/route" + + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" + + "github.com/flomesh-io/fsm/pkg/constants" + gwutils "github.com/flomesh-io/fsm/pkg/gateway/utils" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/record" @@ -36,13 +45,12 @@ import ( fctx "github.com/flomesh-io/fsm/pkg/context" "github.com/flomesh-io/fsm/pkg/controllers" - "github.com/flomesh-io/fsm/pkg/gateway/status" ) type udpRouteReconciler struct { recorder record.EventRecorder fctx *fctx.ControllerContext - statusProcessor *status.RouteStatusProcessor + statusProcessor *route.RouteStatusProcessor } func (r *udpRouteReconciler) NeedLeaderElection() bool { @@ -54,7 +62,7 @@ func NewUDPRouteReconciler(ctx *fctx.ControllerContext) controllers.Reconciler { return &udpRouteReconciler{ recorder: ctx.Manager.GetEventRecorderFor("UDPRoute"), fctx: ctx, - statusProcessor: &status.RouteStatusProcessor{Informers: ctx.InformerCollection}, + statusProcessor: route.NewRouteStatusProcessor(ctx.Manager.GetCache()), } } @@ -76,18 +84,17 @@ func (r *udpRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c return ctrl.Result{}, nil } - routeStatus, err := r.statusProcessor.ProcessRouteStatus(ctx, udpRoute) - if err != nil { + rsu := route.NewRouteStatusUpdate( + udpRoute, + &udpRoute.ObjectMeta, + &udpRoute.TypeMeta, + nil, + gwutils.ToSlicePtr(udpRoute.Status.Parents), + ) + if err := r.statusProcessor.Process(ctx, r.fctx.StatusUpdater, rsu, udpRoute.Spec.ParentRefs); err != nil { return ctrl.Result{}, err } - if len(routeStatus) > 0 { - udpRoute.Status.Parents = routeStatus - if err := r.fctx.Status().Update(ctx, udpRoute); err != nil { - return ctrl.Result{}, err - } - } - r.fctx.GatewayEventHandler.OnAdd(udpRoute, false) return ctrl.Result{}, nil @@ -95,7 +102,55 @@ func (r *udpRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c // SetupWithManager sets up the controller with the Manager. func (r *udpRouteReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). + if err := ctrl.NewControllerManagedBy(mgr). For(&gwv1alpha2.UDPRoute{}). - Complete(r) + Complete(r); err != nil { + return err + } + + return addUDPRouteIndexers(context.Background(), mgr) +} + +func addUDPRouteIndexers(ctx context.Context, mgr manager.Manager) error { + if err := mgr.GetFieldIndexer().IndexField(ctx, &gwv1alpha2.UDPRoute{}, constants.GatewayUDPRouteIndex, func(obj client.Object) []string { + udpRoute := obj.(*gwv1alpha2.UDPRoute) + var gateways []string + for _, parent := range udpRoute.Spec.ParentRefs { + if string(*parent.Kind) == constants.GatewayAPIGatewayKind { + gateways = append(gateways, + types.NamespacedName{ + Namespace: gwutils.NamespaceDerefOr(parent.Namespace, udpRoute.Namespace), + Name: string(parent.Name), + }.String(), + ) + } + } + return gateways + }); err != nil { + return err + } + + if err := mgr.GetFieldIndexer().IndexField(ctx, &gwv1alpha2.UDPRoute{}, constants.BackendUDPRouteIndex, backendUDPRouteIndexFunc); err != nil { + return err + } + return nil +} + +func backendUDPRouteIndexFunc(obj client.Object) []string { + udproute := obj.(*gwv1alpha2.UDPRoute) + var backendRefs []string + for _, rule := range udproute.Spec.Rules { + for _, backend := range rule.BackendRefs { + if backend.Kind == nil || string(*backend.Kind) == constants.KubernetesServiceKind { + backendRefs = append(backendRefs, + types.NamespacedName{ + Namespace: gwutils.NamespaceDerefOr(backend.Namespace, udproute.Namespace), + Name: string(backend.Name), + }.String(), + ) + } + } + } + + return backendRefs } diff --git a/pkg/controllers/gateway/v1beta1/referencegrant_controller.go b/pkg/controllers/gateway/v1beta1/referencegrant_controller.go index e5043d21e..81d84b6d9 100644 --- a/pkg/controllers/gateway/v1beta1/referencegrant_controller.go +++ b/pkg/controllers/gateway/v1beta1/referencegrant_controller.go @@ -27,6 +27,11 @@ package v1beta1 import ( "context" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" + + "github.com/flomesh-io/fsm/pkg/constants" + gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" "k8s.io/apimachinery/pkg/api/errors" @@ -83,7 +88,27 @@ func (r *referenceGrantReconciler) Reconcile(ctx context.Context, req ctrl.Reque // SetupWithManager sets up the controller with the Manager. func (r *referenceGrantReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). + if err := ctrl.NewControllerManagedBy(mgr). For(&gwv1beta1.ReferenceGrant{}). - Complete(r) + Complete(r); err != nil { + return err + } + + return addReferenceGrantIndexers(context.Background(), mgr) +} + +func addReferenceGrantIndexers(ctx context.Context, mgr manager.Manager) error { + if err := mgr.GetFieldIndexer().IndexField(ctx, &gwv1beta1.ReferenceGrant{}, constants.TargetKindRefGrantIndex, func(obj client.Object) []string { + refGrant := obj.(*gwv1beta1.ReferenceGrant) + var referredResources []string + for _, target := range refGrant.Spec.To { + referredResources = append(referredResources, string(target.Kind)) + } + + return referredResources + }); err != nil { + return err + } + + return nil } diff --git a/pkg/controllers/policyattachment/v1alpha1/accesscontrolpolicy_controller.go b/pkg/controllers/policyattachment/v1alpha1/accesscontrolpolicy_controller.go index b3b1a7d16..3092b0dfa 100644 --- a/pkg/controllers/policyattachment/v1alpha1/accesscontrolpolicy_controller.go +++ b/pkg/controllers/policyattachment/v1alpha1/accesscontrolpolicy_controller.go @@ -29,21 +29,24 @@ import ( "fmt" "reflect" + policystatus "github.com/flomesh-io/fsm/pkg/gateway/status/policy" + + "k8s.io/apimachinery/pkg/fields" + + "sigs.k8s.io/controller-runtime/pkg/cache" + + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/handler" gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" gwpkg "github.com/flomesh-io/fsm/pkg/gateway/types" - "github.com/flomesh-io/fsm/pkg/k8s/informers" - - "github.com/flomesh-io/fsm/pkg/gateway/policy/status" "github.com/flomesh-io/fsm/pkg/gateway/policy/utils/accesscontrol" "sigs.k8s.io/controller-runtime/pkg/client" - corev1 "k8s.io/api/core/v1" - - gwtypes "github.com/flomesh-io/fsm/pkg/gateway/types" + "github.com/flomesh-io/fsm/pkg/gateway/status" gwutils "github.com/flomesh-io/fsm/pkg/gateway/utils" metautil "k8s.io/apimachinery/pkg/api/meta" @@ -51,8 +54,6 @@ import ( "k8s.io/apimachinery/pkg/types" gwv1 "sigs.k8s.io/gateway-api/apis/v1" - gwclient "sigs.k8s.io/gateway-api/pkg/client/clientset/versioned" - gwpav1alpha1 "github.com/flomesh-io/fsm/pkg/apis/policyattachment/v1alpha1" "github.com/flomesh-io/fsm/pkg/constants" @@ -64,16 +65,12 @@ import ( fctx "github.com/flomesh-io/fsm/pkg/context" "github.com/flomesh-io/fsm/pkg/controllers" - - policyAttachmentApiClientset "github.com/flomesh-io/fsm/pkg/gen/client/policyattachment/clientset/versioned" ) type accessControlPolicyReconciler struct { - recorder record.EventRecorder - fctx *fctx.ControllerContext - gatewayAPIClient gwclient.Interface - policyAttachmentAPIClient policyAttachmentApiClientset.Interface - statusProcessor *status.PolicyStatusProcessor + recorder record.EventRecorder + fctx *fctx.ControllerContext + statusProcessor *policystatus.PolicyStatusProcessor } func (r *accessControlPolicyReconciler) NeedLeaderElection() bool { @@ -83,13 +80,11 @@ func (r *accessControlPolicyReconciler) NeedLeaderElection() bool { // NewAccessControlPolicyReconciler returns a new AccessControlPolicy Reconciler func NewAccessControlPolicyReconciler(ctx *fctx.ControllerContext) controllers.Reconciler { r := &accessControlPolicyReconciler{ - recorder: ctx.Manager.GetEventRecorderFor("AccessControlPolicy"), - fctx: ctx, - gatewayAPIClient: gwclient.NewForConfigOrDie(ctx.KubeConfig), - policyAttachmentAPIClient: policyAttachmentApiClientset.NewForConfigOrDie(ctx.KubeConfig), + recorder: ctx.Manager.GetEventRecorderFor("AccessControlPolicy"), + fctx: ctx, } - r.statusProcessor = &status.PolicyStatusProcessor{ + r.statusProcessor = &policystatus.PolicyStatusProcessor{ Client: r.fctx.Client, Informer: r.fctx.InformerCollection, GetPolicies: r.getAccessControls, @@ -120,79 +115,78 @@ func (r *accessControlPolicyReconciler) Reconcile(ctx context.Context, req ctrl. return ctrl.Result{}, nil } - metautil.SetStatusCondition( - &policy.Status.Conditions, - r.statusProcessor.Process(ctx, policy, policy.Spec.TargetRef), - ) - if err := r.fctx.Status().Update(ctx, policy); err != nil { - return ctrl.Result{}, err - } + r.statusProcessor.Process(ctx, r.fctx.StatusUpdater, policystatus.NewPolicyUpdate( + policy, + &policy.ObjectMeta, + &policy.TypeMeta, + policy.Spec.TargetRef, + policy.Status.Conditions, + )) r.fctx.GatewayEventHandler.OnAdd(policy, false) return ctrl.Result{}, nil } -func (r *accessControlPolicyReconciler) getAccessControls(policy client.Object, target client.Object) (map[gwpkg.PolicyMatchType][]client.Object, *metav1.Condition) { - accessControlPolicyList, err := r.policyAttachmentAPIClient.GatewayV1alpha1().AccessControlPolicies(corev1.NamespaceAll).List(context.TODO(), metav1.ListOptions{}) - if err != nil { - return nil, status.ConditionPointer(status.InvalidCondition(policy, fmt.Sprintf("Failed to list AccessControlPolicies: %s", err))) - } - +func (r *accessControlPolicyReconciler) getAccessControls(target client.Object) map[gwpkg.PolicyMatchType][]client.Object { + c := r.fctx.Manager.GetCache() policies := make(map[gwpkg.PolicyMatchType][]client.Object) - referenceGrants := r.fctx.InformerCollection.GetGatewayResourcesFromCache(informers.ReferenceGrantResourceType, false) - - for _, p := range accessControlPolicyList.Items { - p := p - if gwutils.IsAcceptedPolicyAttachment(p.Status.Conditions) { - spec := p.Spec - targetRef := spec.TargetRef - - switch { - case gwutils.IsTargetRefToGVK(targetRef, constants.GatewayGVK) && - gwutils.IsRefToTarget(referenceGrants, &p, targetRef, target) && - len(spec.Ports) > 0: - policies[gwpkg.PolicyMatchTypePort] = append(policies[gwpkg.PolicyMatchTypePort], &p) - case (gwutils.IsTargetRefToGVK(targetRef, constants.HTTPRouteGVK) || gwutils.IsTargetRefToGVK(targetRef, constants.GRPCRouteGVK)) && - gwutils.IsRefToTarget(referenceGrants, &p, targetRef, target) && - len(spec.Hostnames) > 0: - policies[gwpkg.PolicyMatchTypeHostnames] = append(policies[gwpkg.PolicyMatchTypeHostnames], &p) - case gwutils.IsTargetRefToGVK(targetRef, constants.HTTPRouteGVK) && - gwutils.IsRefToTarget(referenceGrants, &p, targetRef, target) && - len(spec.HTTPAccessControls) > 0: - policies[gwpkg.PolicyMatchTypeHTTPRoute] = append(policies[gwpkg.PolicyMatchTypeHTTPRoute], &p) - case gwutils.IsTargetRefToGVK(targetRef, constants.GRPCRouteGVK) && - gwutils.IsRefToTarget(referenceGrants, &p, targetRef, target) && - len(spec.GRPCAccessControls) > 0: - policies[gwpkg.PolicyMatchTypeGRPCRoute] = append(policies[gwpkg.PolicyMatchTypeGRPCRoute], &p) - } + + for _, param := range []struct { + matchType gwpkg.PolicyMatchType + fn func(cache.Cache, fields.Selector) []client.Object + selector fields.Selector + }{ + { + matchType: gwpkg.PolicyMatchTypePort, + fn: gwutils.GetAccessControlsMatchTypePort, + selector: fields.OneTermEqualSelector(constants.PortPolicyAttachmentIndex, client.ObjectKeyFromObject(target).String()), + }, + { + matchType: gwpkg.PolicyMatchTypeHostnames, + fn: gwutils.GetAccessControlsMatchTypeHostname, + selector: fields.OneTermEqualSelector(constants.HostnamePolicyAttachmentIndex, fmt.Sprintf("%s/%s/%s", target.GetObjectKind().GroupVersionKind().Kind, target.GetNamespace(), target.GetName())), + }, + { + matchType: gwpkg.PolicyMatchTypeHTTPRoute, + fn: gwutils.GetAccessControlsMatchTypeHTTPRoute, + selector: fields.OneTermEqualSelector(constants.HTTPRoutePolicyAttachmentIndex, client.ObjectKeyFromObject(target).String()), + }, + { + matchType: gwpkg.PolicyMatchTypeGRPCRoute, + fn: gwutils.GetAccessControlsMatchTypeGRPCRoute, + selector: fields.OneTermEqualSelector(constants.GRPCRoutePolicyAttachmentIndex, client.ObjectKeyFromObject(target).String()), + }, + } { + if result := param.fn(c, param.selector); len(result) > 0 { + policies[param.matchType] = result } } - return policies, nil + return policies } -func (r *accessControlPolicyReconciler) getConflictedHostnamesBasedAccessControlPolicy(route *gwtypes.RouteContext, accessControlPolicy client.Object, hostnamesAccessControls []client.Object) *types.NamespacedName { +func (r *accessControlPolicyReconciler) getConflictedHostnamesBasedAccessControlPolicy(route status.RouteStatusObject, parentRefs []gwv1.ParentReference, accessControlPolicy client.Object, hostnamesAccessControls []client.Object) *types.NamespacedName { currentPolicy := accessControlPolicy.(*gwpav1alpha1.AccessControlPolicy) if len(currentPolicy.Spec.Hostnames) == 0 { return nil } - for _, parent := range route.ParentStatus { - if metautil.IsStatusConditionTrue(parent.Conditions, string(gwv1.RouteConditionAccepted)) { - key := getRouteParentKey(route.Meta, parent) + for _, parentRef := range parentRefs { + h := route.StatusUpdateFor(parentRef) + + if metautil.IsStatusConditionTrue(h.ConditionsForParentRef(parentRef), string(gwv1.RouteConditionAccepted)) { + key := getRouteParentKey(route.GetObjectMeta(), parentRef) gateway := &gwv1.Gateway{} if err := r.fctx.Get(context.TODO(), key, gateway); err != nil { continue } - validListeners := gwutils.GetValidListenersForGateway(gateway) - - allowedListeners, _ := gwutils.GetAllowedListeners(r.fctx.InformerCollection.GetListers().Namespace, gateway, parent.ParentRef, route, validListeners) + allowedListeners := gwutils.GetAllowedListeners(r.fctx.Manager.GetCache(), gateway, h) for _, listener := range allowedListeners { - hostnames := gwutils.GetValidHostnames(listener.Hostname, route.Hostnames) + hostnames := gwutils.GetValidHostnames(listener.Hostname, route.GetHostnames()) if len(hostnames) == 0 { // no valid hostnames, should ignore it continue @@ -201,12 +195,12 @@ func (r *accessControlPolicyReconciler) getConflictedHostnamesBasedAccessControl for _, hr := range hostnamesAccessControls { hr := hr.(*gwpav1alpha1.AccessControlPolicy) - r1 := accesscontrol.GetAccessControlConfigIfRouteHostnameMatchesPolicy(hostname, *hr) + r1 := accesscontrol.GetAccessControlConfigIfRouteHostnameMatchesPolicy(hostname, hr) if r1 == nil { continue } - r2 := accesscontrol.GetAccessControlConfigIfRouteHostnameMatchesPolicy(hostname, *currentPolicy) + r2 := accesscontrol.GetAccessControlConfigIfRouteHostnameMatchesPolicy(hostname, currentPolicy) if r2 == nil { continue } @@ -244,12 +238,12 @@ func (r *accessControlPolicyReconciler) getConflictedHTTPRouteBasedAccessControl continue } - r1 := accesscontrol.GetAccessControlConfigIfHTTPRouteMatchesPolicy(m, *routePolicy) + r1 := accesscontrol.GetAccessControlConfigIfHTTPRouteMatchesPolicy(m, routePolicy) if r1 == nil { continue } - r2 := accesscontrol.GetAccessControlConfigIfHTTPRouteMatchesPolicy(m, *currentPolicy) + r2 := accesscontrol.GetAccessControlConfigIfHTTPRouteMatchesPolicy(m, currentPolicy) if r2 == nil { continue } @@ -285,12 +279,12 @@ func (r *accessControlPolicyReconciler) getConflictedGRPCRouteBasedAccessControl continue } - r1 := accesscontrol.GetAccessControlConfigIfGRPCRouteMatchesPolicy(m, *routePolicy) + r1 := accesscontrol.GetAccessControlConfigIfGRPCRouteMatchesPolicy(m, routePolicy) if r1 == nil { continue } - r2 := accesscontrol.GetAccessControlConfigIfGRPCRouteMatchesPolicy(m, *currentPolicy) + r2 := accesscontrol.GetAccessControlConfigIfGRPCRouteMatchesPolicy(m, currentPolicy) if r2 == nil { continue } @@ -323,12 +317,12 @@ func (r *accessControlPolicyReconciler) getConflictedPort(gateway *gwv1.Gateway, if len(accessControl.Spec.Ports) > 0 { for _, listener := range validListeners { - r1 := accesscontrol.GetAccessControlConfigIfPortMatchesPolicy(listener.Port, *accessControl) + r1 := accesscontrol.GetAccessControlConfigIfPortMatchesPolicy(listener.Port, accessControl) if r1 == nil { continue } - r2 := accesscontrol.GetAccessControlConfigIfPortMatchesPolicy(listener.Port, *currentPolicy) + r2 := accesscontrol.GetAccessControlConfigIfPortMatchesPolicy(listener.Port, currentPolicy) if r2 == nil { continue } @@ -350,13 +344,94 @@ func (r *accessControlPolicyReconciler) getConflictedPort(gateway *gwv1.Gateway, // SetupWithManager sets up the controller with the Manager. func (r *accessControlPolicyReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). + if err := ctrl.NewControllerManagedBy(mgr). For(&gwpav1alpha1.AccessControlPolicy{}). Watches( &gwv1beta1.ReferenceGrant{}, handler.EnqueueRequestsFromMapFunc(r.referenceGrantToPolicyAttachment), ). - Complete(r) + Complete(r); err != nil { + return err + } + + return addAccessControlPolicyIndexer(context.Background(), mgr) +} + +func addAccessControlPolicyIndexer(ctx context.Context, mgr manager.Manager) error { + if err := mgr.GetFieldIndexer().IndexField(ctx, &gwpav1alpha1.AccessControlPolicy{}, constants.PortPolicyAttachmentIndex, addAccessControlPortIndexFunc); err != nil { + return err + } + + if err := mgr.GetFieldIndexer().IndexField(ctx, &gwpav1alpha1.AccessControlPolicy{}, constants.HostnamePolicyAttachmentIndex, addAccessControlHostnameIndexFunc); err != nil { + return err + } + + if err := mgr.GetFieldIndexer().IndexField(ctx, &gwpav1alpha1.AccessControlPolicy{}, constants.HTTPRoutePolicyAttachmentIndex, addAccessControlHTTPRouteIndexFunc); err != nil { + return err + } + + if err := mgr.GetFieldIndexer().IndexField(ctx, &gwpav1alpha1.AccessControlPolicy{}, constants.GRPCRoutePolicyAttachmentIndex, addAccessControlGRPCRouteIndexFunc); err != nil { + return err + } + + return nil +} + +func addAccessControlPortIndexFunc(obj client.Object) []string { + policy := obj.(*gwpav1alpha1.AccessControlPolicy) + targetRef := policy.Spec.TargetRef + + var targets []string + if gwutils.IsTargetRefToGVK(targetRef, constants.GatewayGVK) && len(policy.Spec.Ports) > 0 { + targets = append(targets, types.NamespacedName{ + Namespace: gwutils.NamespaceDerefOr(targetRef.Namespace, policy.Namespace), + Name: string(targetRef.Name), + }.String()) + } + + return targets +} + +func addAccessControlHostnameIndexFunc(obj client.Object) []string { + policy := obj.(*gwpav1alpha1.AccessControlPolicy) + targetRef := policy.Spec.TargetRef + + var targets []string + if (gwutils.IsTargetRefToGVK(targetRef, constants.HTTPRouteGVK) || gwutils.IsTargetRefToGVK(targetRef, constants.GRPCRouteGVK)) && len(policy.Spec.Hostnames) > 0 { + targets = append(targets, fmt.Sprintf("%s/%s/%s", targetRef.Kind, gwutils.NamespaceDerefOr(targetRef.Namespace, policy.Namespace), string(targetRef.Name))) + } + + return targets +} + +func addAccessControlHTTPRouteIndexFunc(obj client.Object) []string { + policy := obj.(*gwpav1alpha1.AccessControlPolicy) + targetRef := policy.Spec.TargetRef + + var targets []string + if gwutils.IsTargetRefToGVK(targetRef, constants.HTTPRouteGVK) && len(policy.Spec.HTTPAccessControls) > 0 { + targets = append(targets, types.NamespacedName{ + Namespace: gwutils.NamespaceDerefOr(targetRef.Namespace, policy.Namespace), + Name: string(targetRef.Name), + }.String()) + } + + return targets +} + +func addAccessControlGRPCRouteIndexFunc(obj client.Object) []string { + policy := obj.(*gwpav1alpha1.AccessControlPolicy) + targetRef := policy.Spec.TargetRef + + var targets []string + if gwutils.IsTargetRefToGVK(targetRef, constants.GRPCRouteGVK) && len(policy.Spec.GRPCAccessControls) > 0 { + targets = append(targets, types.NamespacedName{ + Namespace: gwutils.NamespaceDerefOr(targetRef.Namespace, policy.Namespace), + Name: string(targetRef.Name), + }.String()) + } + + return targets } func (r *accessControlPolicyReconciler) referenceGrantToPolicyAttachment(_ context.Context, obj client.Object) []reconcile.Request { @@ -366,13 +441,17 @@ func (r *accessControlPolicyReconciler) referenceGrantToPolicyAttachment(_ conte return nil } - requests := make([]reconcile.Request, 0) - policies := r.fctx.InformerCollection.GetGatewayResourcesFromCache(informers.AccessControlPoliciesResourceType, false) - - for _, p := range policies { - policy := p.(*gwpav1alpha1.AccessControlPolicy) + c := r.fctx.Manager.GetCache() + list := &gwpav1alpha1.AccessControlPolicyList{} + if err := c.List(context.Background(), list); err != nil { + log.Error().Msgf("Failed to list AccessControlPolicies: %v", err) + return nil + } + policies := gwutils.ToSlicePtr(list.Items) - if gwutils.HasAccessToTargetRef(policy, policy.Spec.TargetRef, []client.Object{refGrant}) { + requests := make([]reconcile.Request, 0) + for _, policy := range policies { + if gwutils.HasAccessToTargetRef(policy, policy.Spec.TargetRef, []*gwv1beta1.ReferenceGrant{refGrant}) { requests = append(requests, reconcile.Request{ NamespacedName: types.NamespacedName{ Name: policy.Name, diff --git a/pkg/controllers/policyattachment/v1alpha1/circuitbreakingpolicy_controller.go b/pkg/controllers/policyattachment/v1alpha1/circuitbreakingpolicy_controller.go index f0c909e41..0bffe09cc 100644 --- a/pkg/controllers/policyattachment/v1alpha1/circuitbreakingpolicy_controller.go +++ b/pkg/controllers/policyattachment/v1alpha1/circuitbreakingpolicy_controller.go @@ -2,15 +2,17 @@ package v1alpha1 import ( "context" - "fmt" "reflect" - "sigs.k8s.io/controller-runtime/pkg/handler" - gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" + policystatus "github.com/flomesh-io/fsm/pkg/gateway/status/policy" - "github.com/flomesh-io/fsm/pkg/k8s/informers" + "k8s.io/apimachinery/pkg/fields" - "github.com/flomesh-io/fsm/pkg/gateway/policy/status" + "github.com/flomesh-io/fsm/pkg/constants" + + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" "github.com/flomesh-io/fsm/pkg/gateway/policy/utils/circuitbreaking" @@ -20,12 +22,6 @@ import ( "k8s.io/apimachinery/pkg/types" - corev1 "k8s.io/api/core/v1" - - metautil "k8s.io/apimachinery/pkg/api/meta" - - gwclient "sigs.k8s.io/gateway-api/pkg/client/clientset/versioned" - "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/record" @@ -36,16 +32,12 @@ import ( fctx "github.com/flomesh-io/fsm/pkg/context" "github.com/flomesh-io/fsm/pkg/controllers" - - policyAttachmentApiClientset "github.com/flomesh-io/fsm/pkg/gen/client/policyattachment/clientset/versioned" ) type circuitBreakingPolicyReconciler struct { - recorder record.EventRecorder - fctx *fctx.ControllerContext - gatewayAPIClient gwclient.Interface - policyAttachmentAPIClient policyAttachmentApiClientset.Interface - statusProcessor *status.ServicePolicyStatusProcessor + recorder record.EventRecorder + fctx *fctx.ControllerContext + statusProcessor *policystatus.ServicePolicyStatusProcessor } func (r *circuitBreakingPolicyReconciler) NeedLeaderElection() bool { @@ -55,13 +47,11 @@ func (r *circuitBreakingPolicyReconciler) NeedLeaderElection() bool { // NewCircuitBreakingPolicyReconciler returns a new CircuitBreakingPolicy Reconciler func NewCircuitBreakingPolicyReconciler(ctx *fctx.ControllerContext) controllers.Reconciler { r := &circuitBreakingPolicyReconciler{ - recorder: ctx.Manager.GetEventRecorderFor("CircuitBreakingPolicy"), - fctx: ctx, - gatewayAPIClient: gwclient.NewForConfigOrDie(ctx.KubeConfig), - policyAttachmentAPIClient: policyAttachmentApiClientset.NewForConfigOrDie(ctx.KubeConfig), + recorder: ctx.Manager.GetEventRecorderFor("CircuitBreakingPolicy"), + fctx: ctx, } - r.statusProcessor = &status.ServicePolicyStatusProcessor{ + r.statusProcessor = &policystatus.ServicePolicyStatusProcessor{ Client: r.fctx.Client, Informer: r.fctx.InformerCollection, GetAttachedPolicies: r.getAttachedCircuitBreakings, @@ -89,37 +79,25 @@ func (r *circuitBreakingPolicyReconciler) Reconcile(ctx context.Context, req ctr return ctrl.Result{}, nil } - metautil.SetStatusCondition( - &policy.Status.Conditions, - r.statusProcessor.Process(ctx, policy, policy.Spec.TargetRef), - ) - if err := r.fctx.Status().Update(ctx, policy); err != nil { - return ctrl.Result{}, err - } + r.statusProcessor.Process(ctx, r.fctx.StatusUpdater, policystatus.NewPolicyUpdate( + policy, + &policy.ObjectMeta, + &policy.TypeMeta, + policy.Spec.TargetRef, + policy.Status.Conditions, + )) r.fctx.GatewayEventHandler.OnAdd(policy, false) return ctrl.Result{}, nil } -func (r *circuitBreakingPolicyReconciler) getAttachedCircuitBreakings(policy client.Object, svc client.Object) ([]client.Object, *metav1.Condition) { - circuitBreakingPolicyList, err := r.policyAttachmentAPIClient.GatewayV1alpha1().CircuitBreakingPolicies(corev1.NamespaceAll).List(context.TODO(), metav1.ListOptions{}) - if err != nil { - return nil, status.ConditionPointer(status.InvalidCondition(policy, fmt.Sprintf("Failed to list CircuitBreakingPolicies: %s", err))) - } +func (r *circuitBreakingPolicyReconciler) getAttachedCircuitBreakings(svc client.Object) ([]client.Object, *metav1.Condition) { + c := r.fctx.Manager.GetCache() + key := client.ObjectKeyFromObject(svc).String() + selector := fields.OneTermEqualSelector(constants.ServicePolicyAttachmentIndex, key) - circuitBreakings := make([]client.Object, 0) - referenceGrants := r.fctx.InformerCollection.GetGatewayResourcesFromCache(informers.ReferenceGrantResourceType, false) - - for _, p := range circuitBreakingPolicyList.Items { - p := p - if gwutils.IsAcceptedPolicyAttachment(p.Status.Conditions) && - gwutils.IsRefToTarget(referenceGrants, &p, p.Spec.TargetRef, svc) { - circuitBreakings = append(circuitBreakings, &p) - } - } - - return circuitBreakings, nil + return gwutils.GetCircuitBreakings(c, selector), nil } func (r *circuitBreakingPolicyReconciler) findConflict(circuitBreakingPolicy client.Object, allCircuitBreakingPolicies []client.Object, port int32) *types.NamespacedName { @@ -153,13 +131,37 @@ func (r *circuitBreakingPolicyReconciler) findConflict(circuitBreakingPolicy cli // SetupWithManager sets up the controller with the Manager. func (r *circuitBreakingPolicyReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). + if err := ctrl.NewControllerManagedBy(mgr). For(&gwpav1alpha1.CircuitBreakingPolicy{}). Watches( &gwv1beta1.ReferenceGrant{}, handler.EnqueueRequestsFromMapFunc(r.referenceGrantToPolicyAttachment), ). - Complete(r) + Complete(r); err != nil { + return err + } + + return addCircuitBreakingPolicyIndexer(context.Background(), mgr) +} + +func addCircuitBreakingPolicyIndexer(ctx context.Context, mgr manager.Manager) error { + if err := mgr.GetFieldIndexer().IndexField(ctx, &gwpav1alpha1.CircuitBreakingPolicy{}, constants.ServicePolicyAttachmentIndex, func(obj client.Object) []string { + policy := obj.(*gwpav1alpha1.CircuitBreakingPolicy) + targetRef := policy.Spec.TargetRef + var targets []string + if targetRef.Kind == constants.KubernetesServiceKind { + targets = append(targets, types.NamespacedName{ + Namespace: gwutils.NamespaceDerefOr(targetRef.Namespace, policy.Namespace), + Name: string(targetRef.Name), + }.String()) + } + + return targets + }); err != nil { + return err + } + + return nil } func (r *circuitBreakingPolicyReconciler) referenceGrantToPolicyAttachment(_ context.Context, obj client.Object) []reconcile.Request { @@ -169,13 +171,21 @@ func (r *circuitBreakingPolicyReconciler) referenceGrantToPolicyAttachment(_ con return nil } + c := r.fctx.Manager.GetCache() + list := &gwpav1alpha1.CircuitBreakingPolicyList{} + if err := c.List(context.Background(), list); err != nil { + log.Error().Msgf("Failed to list CircuitBreakingPolicyList: %v", err) + return nil + } + policies := gwutils.ToSlicePtr(list.Items) + requests := make([]reconcile.Request, 0) - policies := r.fctx.InformerCollection.GetGatewayResourcesFromCache(informers.CircuitBreakingPoliciesResourceType, false) + //policies := r.fctx.InformerCollection.GetGatewayResourcesFromCache(informers.CircuitBreakingPoliciesResourceType, false) - for _, p := range policies { - policy := p.(*gwpav1alpha1.CircuitBreakingPolicy) + for _, policy := range policies { + //policy := p.(*gwpav1alpha1.CircuitBreakingPolicy) - if gwutils.HasAccessToTargetRef(policy, policy.Spec.TargetRef, []client.Object{refGrant}) { + if gwutils.HasAccessToTargetRef(policy, policy.Spec.TargetRef, []*gwv1beta1.ReferenceGrant{refGrant}) { requests = append(requests, reconcile.Request{ NamespacedName: types.NamespacedName{ Name: policy.Name, diff --git a/pkg/controllers/policyattachment/v1alpha1/faultinjectionpolicy_controller.go b/pkg/controllers/policyattachment/v1alpha1/faultinjectionpolicy_controller.go index a5c7029b5..cc0e5035a 100644 --- a/pkg/controllers/policyattachment/v1alpha1/faultinjectionpolicy_controller.go +++ b/pkg/controllers/policyattachment/v1alpha1/faultinjectionpolicy_controller.go @@ -29,22 +29,25 @@ import ( "fmt" "reflect" + policystatus "github.com/flomesh-io/fsm/pkg/gateway/status/policy" + + "k8s.io/apimachinery/pkg/fields" + + "github.com/flomesh-io/fsm/pkg/gateway/status" + + "sigs.k8s.io/controller-runtime/pkg/cache" + + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/handler" gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" - "github.com/flomesh-io/fsm/pkg/k8s/informers" - gwpkg "github.com/flomesh-io/fsm/pkg/gateway/types" - "github.com/flomesh-io/fsm/pkg/gateway/policy/status" - "github.com/flomesh-io/fsm/pkg/gateway/policy/utils/faultinjection" "sigs.k8s.io/controller-runtime/pkg/client" - corev1 "k8s.io/api/core/v1" - - gwtypes "github.com/flomesh-io/fsm/pkg/gateway/types" gwutils "github.com/flomesh-io/fsm/pkg/gateway/utils" metautil "k8s.io/apimachinery/pkg/api/meta" @@ -52,8 +55,6 @@ import ( "k8s.io/apimachinery/pkg/types" gwv1 "sigs.k8s.io/gateway-api/apis/v1" - gwclient "sigs.k8s.io/gateway-api/pkg/client/clientset/versioned" - gwpav1alpha1 "github.com/flomesh-io/fsm/pkg/apis/policyattachment/v1alpha1" "github.com/flomesh-io/fsm/pkg/constants" @@ -65,16 +66,12 @@ import ( fctx "github.com/flomesh-io/fsm/pkg/context" "github.com/flomesh-io/fsm/pkg/controllers" - - policyAttachmentApiClientset "github.com/flomesh-io/fsm/pkg/gen/client/policyattachment/clientset/versioned" ) type faultInjectionPolicyReconciler struct { - recorder record.EventRecorder - fctx *fctx.ControllerContext - gatewayAPIClient gwclient.Interface - policyAttachmentAPIClient policyAttachmentApiClientset.Interface - statusProcessor *status.PolicyStatusProcessor + recorder record.EventRecorder + fctx *fctx.ControllerContext + statusProcessor *policystatus.PolicyStatusProcessor } func (r *faultInjectionPolicyReconciler) NeedLeaderElection() bool { @@ -84,13 +81,11 @@ func (r *faultInjectionPolicyReconciler) NeedLeaderElection() bool { // NewFaultInjectionPolicyReconciler returns a new FaultInjectionPolicy Reconciler func NewFaultInjectionPolicyReconciler(ctx *fctx.ControllerContext) controllers.Reconciler { r := &faultInjectionPolicyReconciler{ - recorder: ctx.Manager.GetEventRecorderFor("FaultInjectionPolicy"), - fctx: ctx, - gatewayAPIClient: gwclient.NewForConfigOrDie(ctx.KubeConfig), - policyAttachmentAPIClient: policyAttachmentApiClientset.NewForConfigOrDie(ctx.KubeConfig), + recorder: ctx.Manager.GetEventRecorderFor("FaultInjectionPolicy"), + fctx: ctx, } - r.statusProcessor = &status.PolicyStatusProcessor{ + r.statusProcessor = &policystatus.PolicyStatusProcessor{ Client: r.fctx.Client, Informer: r.fctx.InformerCollection, GetPolicies: r.getFaultInjections, @@ -126,75 +121,73 @@ func (r *faultInjectionPolicyReconciler) Reconcile(ctx context.Context, req ctrl return ctrl.Result{}, nil } - metautil.SetStatusCondition( - &policy.Status.Conditions, - r.statusProcessor.Process(ctx, policy, policy.Spec.TargetRef), - ) - if err := r.fctx.Status().Update(ctx, policy); err != nil { - return ctrl.Result{}, err - } + r.statusProcessor.Process(ctx, r.fctx.StatusUpdater, policystatus.NewPolicyUpdate( + policy, + &policy.ObjectMeta, + &policy.TypeMeta, + policy.Spec.TargetRef, + policy.Status.Conditions, + )) r.fctx.GatewayEventHandler.OnAdd(policy, false) return ctrl.Result{}, nil } -func (r *faultInjectionPolicyReconciler) getFaultInjections(policy client.Object, target client.Object) (map[gwpkg.PolicyMatchType][]client.Object, *metav1.Condition) { - faultInjectionPolicyList, err := r.policyAttachmentAPIClient.GatewayV1alpha1().FaultInjectionPolicies(corev1.NamespaceAll).List(context.TODO(), metav1.ListOptions{}) - if err != nil { - return nil, status.ConditionPointer(status.InvalidCondition(policy, fmt.Sprintf("Failed to list FaultInjectionPolicies: %s", err))) - } - +func (r *faultInjectionPolicyReconciler) getFaultInjections(target client.Object) map[gwpkg.PolicyMatchType][]client.Object { + c := r.fctx.Manager.GetCache() policies := make(map[gwpkg.PolicyMatchType][]client.Object) - referenceGrants := r.fctx.InformerCollection.GetGatewayResourcesFromCache(informers.ReferenceGrantResourceType, false) - - for _, p := range faultInjectionPolicyList.Items { - p := p - if gwutils.IsAcceptedPolicyAttachment(p.Status.Conditions) { - spec := p.Spec - targetRef := spec.TargetRef - - switch { - case (gwutils.IsTargetRefToGVK(targetRef, constants.HTTPRouteGVK) || gwutils.IsTargetRefToGVK(targetRef, constants.GRPCRouteGVK)) && - gwutils.IsRefToTarget(referenceGrants, &p, targetRef, target) && - len(spec.Hostnames) > 0: - policies[gwpkg.PolicyMatchTypeHostnames] = append(policies[gwpkg.PolicyMatchTypeHostnames], &p) - case gwutils.IsTargetRefToGVK(targetRef, constants.HTTPRouteGVK) && - gwutils.IsRefToTarget(referenceGrants, &p, targetRef, target) && - len(spec.HTTPFaultInjections) > 0: - policies[gwpkg.PolicyMatchTypeHTTPRoute] = append(policies[gwpkg.PolicyMatchTypeHTTPRoute], &p) - case gwutils.IsTargetRefToGVK(targetRef, constants.GRPCRouteGVK) && - gwutils.IsRefToTarget(referenceGrants, &p, targetRef, target) && - len(spec.GRPCFaultInjections) > 0: - policies[gwpkg.PolicyMatchTypeGRPCRoute] = append(policies[gwpkg.PolicyMatchTypeGRPCRoute], &p) - } + + for _, param := range []struct { + matchType gwpkg.PolicyMatchType + fn func(cache.Cache, fields.Selector) []client.Object + selector fields.Selector + }{ + { + matchType: gwpkg.PolicyMatchTypeHostnames, + fn: gwutils.GetFaultInjectionsMatchTypeHostname, + selector: fields.OneTermEqualSelector(constants.HostnamePolicyAttachmentIndex, fmt.Sprintf("%s/%s/%s", target.GetObjectKind().GroupVersionKind().Kind, target.GetNamespace(), target.GetName())), + }, + { + matchType: gwpkg.PolicyMatchTypeHTTPRoute, + fn: gwutils.GetFaultInjectionsMatchTypeHTTPRoute, + selector: fields.OneTermEqualSelector(constants.HTTPRoutePolicyAttachmentIndex, client.ObjectKeyFromObject(target).String()), + }, + { + matchType: gwpkg.PolicyMatchTypeGRPCRoute, + fn: gwutils.GetFaultInjectionsMatchTypeGRPCRoute, + selector: fields.OneTermEqualSelector(constants.GRPCRoutePolicyAttachmentIndex, client.ObjectKeyFromObject(target).String()), + }, + } { + if result := param.fn(c, param.selector); len(result) > 0 { + policies[param.matchType] = result } } - return policies, nil + return policies } -func (r *faultInjectionPolicyReconciler) getConflictedHostnamesBasedFaultInjectionPolicy(route *gwtypes.RouteContext, faultInjectionPolicy client.Object, hostnamesFaultInjections []client.Object) *types.NamespacedName { +func (r *faultInjectionPolicyReconciler) getConflictedHostnamesBasedFaultInjectionPolicy(route status.RouteStatusObject, parentRefs []gwv1.ParentReference, faultInjectionPolicy client.Object, hostnamesFaultInjections []client.Object) *types.NamespacedName { currentPolicy := faultInjectionPolicy.(*gwpav1alpha1.FaultInjectionPolicy) if len(currentPolicy.Spec.Hostnames) == 0 { return nil } - for _, parent := range route.ParentStatus { - if metautil.IsStatusConditionTrue(parent.Conditions, string(gwv1.RouteConditionAccepted)) { - key := getRouteParentKey(route.Meta, parent) + for _, parentRef := range parentRefs { + h := route.StatusUpdateFor(parentRef) + + if metautil.IsStatusConditionTrue(h.ConditionsForParentRef(parentRef), string(gwv1.RouteConditionAccepted)) { + key := getRouteParentKey(route.GetObjectMeta(), parentRef) gateway := &gwv1.Gateway{} if err := r.fctx.Get(context.TODO(), key, gateway); err != nil { continue } - validListeners := gwutils.GetValidListenersForGateway(gateway) - - allowedListeners, _ := gwutils.GetAllowedListeners(r.fctx.InformerCollection.GetListers().Namespace, gateway, parent.ParentRef, route, validListeners) + allowedListeners := gwutils.GetAllowedListeners(r.fctx.Manager.GetCache(), gateway, h) for _, listener := range allowedListeners { - hostnames := gwutils.GetValidHostnames(listener.Hostname, route.Hostnames) + hostnames := gwutils.GetValidHostnames(listener.Hostname, route.GetHostnames()) if len(hostnames) == 0 { // no valid hostnames, should ignore it continue @@ -203,12 +196,12 @@ func (r *faultInjectionPolicyReconciler) getConflictedHostnamesBasedFaultInjecti for _, hr := range hostnamesFaultInjections { hr := hr.(*gwpav1alpha1.FaultInjectionPolicy) - r1 := faultinjection.GetFaultInjectionConfigIfRouteHostnameMatchesPolicy(hostname, *hr) + r1 := faultinjection.GetFaultInjectionConfigIfRouteHostnameMatchesPolicy(hostname, hr) if r1 == nil { continue } - r2 := faultinjection.GetFaultInjectionConfigIfRouteHostnameMatchesPolicy(hostname, *currentPolicy) + r2 := faultinjection.GetFaultInjectionConfigIfRouteHostnameMatchesPolicy(hostname, currentPolicy) if r2 == nil { continue } @@ -246,12 +239,12 @@ func (r *faultInjectionPolicyReconciler) getConflictedHTTPRouteBasedFaultInjecti continue } - r1 := faultinjection.GetFaultInjectionConfigIfHTTPRouteMatchesPolicy(m, *faultInjection) + r1 := faultinjection.GetFaultInjectionConfigIfHTTPRouteMatchesPolicy(m, faultInjection) if r1 == nil { continue } - r2 := faultinjection.GetFaultInjectionConfigIfHTTPRouteMatchesPolicy(m, *currentPolicy) + r2 := faultinjection.GetFaultInjectionConfigIfHTTPRouteMatchesPolicy(m, currentPolicy) if r2 == nil { continue } @@ -287,12 +280,12 @@ func (r *faultInjectionPolicyReconciler) getConflictedGRPCRouteBasedRFaultInject continue } - r1 := faultinjection.GetFaultInjectionConfigIfGRPCRouteMatchesPolicy(m, *rr) + r1 := faultinjection.GetFaultInjectionConfigIfGRPCRouteMatchesPolicy(m, rr) if r1 == nil { continue } - r2 := faultinjection.GetFaultInjectionConfigIfGRPCRouteMatchesPolicy(m, *currentPolicy) + r2 := faultinjection.GetFaultInjectionConfigIfGRPCRouteMatchesPolicy(m, currentPolicy) if r2 == nil { continue } @@ -314,13 +307,75 @@ func (r *faultInjectionPolicyReconciler) getConflictedGRPCRouteBasedRFaultInject // SetupWithManager sets up the controller with the Manager. func (r *faultInjectionPolicyReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). + if err := ctrl.NewControllerManagedBy(mgr). For(&gwpav1alpha1.FaultInjectionPolicy{}). Watches( &gwv1beta1.ReferenceGrant{}, handler.EnqueueRequestsFromMapFunc(r.referenceGrantToPolicyAttachment), ). - Complete(r) + Complete(r); err != nil { + return err + } + + return addFaultInjectionPolicyIndexer(context.Background(), mgr) +} + +func addFaultInjectionPolicyIndexer(ctx context.Context, mgr manager.Manager) error { + if err := mgr.GetFieldIndexer().IndexField(ctx, &gwpav1alpha1.FaultInjectionPolicy{}, constants.HostnamePolicyAttachmentIndex, addFaultInjectionHostnameIndexFunc); err != nil { + return err + } + + if err := mgr.GetFieldIndexer().IndexField(ctx, &gwpav1alpha1.FaultInjectionPolicy{}, constants.HTTPRoutePolicyAttachmentIndex, addFaultInjectionHTTPRouteIndexFunc); err != nil { + return err + } + + if err := mgr.GetFieldIndexer().IndexField(ctx, &gwpav1alpha1.FaultInjectionPolicy{}, constants.GRPCRoutePolicyAttachmentIndex, addFaultInjectionGRPCRouteIndexFunc); err != nil { + return err + } + + return nil +} + +func addFaultInjectionHostnameIndexFunc(obj client.Object) []string { + policy := obj.(*gwpav1alpha1.FaultInjectionPolicy) + targetRef := policy.Spec.TargetRef + + var targets []string + if (gwutils.IsTargetRefToGVK(targetRef, constants.HTTPRouteGVK) || gwutils.IsTargetRefToGVK(targetRef, constants.GRPCRouteGVK)) && len(policy.Spec.Hostnames) > 0 { + targets = append(targets, fmt.Sprintf("%s/%s/%s", targetRef.Kind, gwutils.NamespaceDerefOr(targetRef.Namespace, policy.Namespace), string(targetRef.Name))) + } + + return targets +} + +func addFaultInjectionHTTPRouteIndexFunc(obj client.Object) []string { + policy := obj.(*gwpav1alpha1.FaultInjectionPolicy) + targetRef := policy.Spec.TargetRef + + var targets []string + if gwutils.IsTargetRefToGVK(targetRef, constants.HTTPRouteGVK) && len(policy.Spec.HTTPFaultInjections) > 0 { + targets = append(targets, types.NamespacedName{ + Namespace: gwutils.NamespaceDerefOr(targetRef.Namespace, policy.Namespace), + Name: string(targetRef.Name), + }.String()) + } + + return targets +} + +func addFaultInjectionGRPCRouteIndexFunc(obj client.Object) []string { + policy := obj.(*gwpav1alpha1.FaultInjectionPolicy) + targetRef := policy.Spec.TargetRef + + var targets []string + if gwutils.IsTargetRefToGVK(targetRef, constants.GRPCRouteGVK) && len(policy.Spec.GRPCFaultInjections) > 0 { + targets = append(targets, types.NamespacedName{ + Namespace: gwutils.NamespaceDerefOr(targetRef.Namespace, policy.Namespace), + Name: string(targetRef.Name), + }.String()) + } + + return targets } func (r *faultInjectionPolicyReconciler) referenceGrantToPolicyAttachment(_ context.Context, obj client.Object) []reconcile.Request { @@ -330,13 +385,17 @@ func (r *faultInjectionPolicyReconciler) referenceGrantToPolicyAttachment(_ cont return nil } - requests := make([]reconcile.Request, 0) - policies := r.fctx.InformerCollection.GetGatewayResourcesFromCache(informers.FaultInjectionPoliciesResourceType, false) - - for _, p := range policies { - policy := p.(*gwpav1alpha1.FaultInjectionPolicy) + c := r.fctx.Manager.GetCache() + list := &gwpav1alpha1.FaultInjectionPolicyList{} + if err := c.List(context.Background(), list); err != nil { + log.Error().Msgf("Failed to list FaultInjectionPolicyList: %v", err) + return nil + } + policies := gwutils.ToSlicePtr(list.Items) - if gwutils.HasAccessToTargetRef(policy, policy.Spec.TargetRef, []client.Object{refGrant}) { + requests := make([]reconcile.Request, 0) + for _, policy := range policies { + if gwutils.HasAccessToTargetRef(policy, policy.Spec.TargetRef, []*gwv1beta1.ReferenceGrant{refGrant}) { requests = append(requests, reconcile.Request{ NamespacedName: types.NamespacedName{ Name: policy.Name, diff --git a/pkg/controllers/policyattachment/v1alpha1/healthcheckpolicy_controller.go b/pkg/controllers/policyattachment/v1alpha1/healthcheckpolicy_controller.go index 53d3debed..2c453ac70 100644 --- a/pkg/controllers/policyattachment/v1alpha1/healthcheckpolicy_controller.go +++ b/pkg/controllers/policyattachment/v1alpha1/healthcheckpolicy_controller.go @@ -2,15 +2,18 @@ package v1alpha1 import ( "context" - "fmt" "reflect" - "sigs.k8s.io/controller-runtime/pkg/handler" - gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" + policystatus "github.com/flomesh-io/fsm/pkg/gateway/status/policy" - "github.com/flomesh-io/fsm/pkg/k8s/informers" + "k8s.io/apimachinery/pkg/fields" - "github.com/flomesh-io/fsm/pkg/gateway/policy/status" + "sigs.k8s.io/controller-runtime/pkg/manager" + + "github.com/flomesh-io/fsm/pkg/constants" + + "sigs.k8s.io/controller-runtime/pkg/handler" + gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" "github.com/flomesh-io/fsm/pkg/gateway/policy/utils/healthcheck" @@ -20,12 +23,6 @@ import ( "k8s.io/apimachinery/pkg/types" - corev1 "k8s.io/api/core/v1" - - metautil "k8s.io/apimachinery/pkg/api/meta" - - gwclient "sigs.k8s.io/gateway-api/pkg/client/clientset/versioned" - "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/record" @@ -36,16 +33,12 @@ import ( fctx "github.com/flomesh-io/fsm/pkg/context" "github.com/flomesh-io/fsm/pkg/controllers" - - policyAttachmentApiClientset "github.com/flomesh-io/fsm/pkg/gen/client/policyattachment/clientset/versioned" ) type healthCheckPolicyReconciler struct { - recorder record.EventRecorder - fctx *fctx.ControllerContext - gatewayAPIClient gwclient.Interface - policyAttachmentAPIClient policyAttachmentApiClientset.Interface - statusProcessor *status.ServicePolicyStatusProcessor + recorder record.EventRecorder + fctx *fctx.ControllerContext + statusProcessor *policystatus.ServicePolicyStatusProcessor } func (r *healthCheckPolicyReconciler) NeedLeaderElection() bool { @@ -55,13 +48,11 @@ func (r *healthCheckPolicyReconciler) NeedLeaderElection() bool { // NewHealthCheckPolicyReconciler returns a new HealthCheckPolicy Reconciler func NewHealthCheckPolicyReconciler(ctx *fctx.ControllerContext) controllers.Reconciler { r := &healthCheckPolicyReconciler{ - recorder: ctx.Manager.GetEventRecorderFor("HealthCheckPolicy"), - fctx: ctx, - gatewayAPIClient: gwclient.NewForConfigOrDie(ctx.KubeConfig), - policyAttachmentAPIClient: policyAttachmentApiClientset.NewForConfigOrDie(ctx.KubeConfig), + recorder: ctx.Manager.GetEventRecorderFor("HealthCheckPolicy"), + fctx: ctx, } - r.statusProcessor = &status.ServicePolicyStatusProcessor{ + r.statusProcessor = &policystatus.ServicePolicyStatusProcessor{ Client: r.fctx.Client, Informer: r.fctx.InformerCollection, GetAttachedPolicies: r.getAttachedHealthChecks, @@ -89,13 +80,13 @@ func (r *healthCheckPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Re return ctrl.Result{}, nil } - metautil.SetStatusCondition( - &policy.Status.Conditions, - r.statusProcessor.Process(ctx, policy, policy.Spec.TargetRef), - ) - if err := r.fctx.Status().Update(ctx, policy); err != nil { - return ctrl.Result{}, err - } + r.statusProcessor.Process(ctx, r.fctx.StatusUpdater, policystatus.NewPolicyUpdate( + policy, + &policy.ObjectMeta, + &policy.TypeMeta, + policy.Spec.TargetRef, + policy.Status.Conditions, + )) r.fctx.GatewayEventHandler.OnAdd(policy, false) @@ -104,32 +95,45 @@ func (r *healthCheckPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Re // SetupWithManager sets up the controller with the Manager. func (r *healthCheckPolicyReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). + if err := ctrl.NewControllerManagedBy(mgr). For(&gwpav1alpha1.HealthCheckPolicy{}). Watches( &gwv1beta1.ReferenceGrant{}, handler.EnqueueRequestsFromMapFunc(r.referenceGrantToPolicyAttachment), ). - Complete(r) -} - -func (r *healthCheckPolicyReconciler) getAttachedHealthChecks(policy client.Object, svc client.Object) ([]client.Object, *metav1.Condition) { - healthCheckPolicyList, err := r.policyAttachmentAPIClient.GatewayV1alpha1().HealthCheckPolicies(corev1.NamespaceAll).List(context.TODO(), metav1.ListOptions{}) - if err != nil { - return nil, status.ConditionPointer(status.InvalidCondition(policy, fmt.Sprintf("Failed to list HealthCheckPolicies: %s", err))) + Complete(r); err != nil { + return err } - referenceGrants := r.fctx.InformerCollection.GetGatewayResourcesFromCache(informers.ReferenceGrantResourceType, false) - healthChecks := make([]client.Object, 0) - for _, p := range healthCheckPolicyList.Items { - p := p - if gwutils.IsAcceptedPolicyAttachment(p.Status.Conditions) && - gwutils.IsRefToTarget(referenceGrants, &p, p.Spec.TargetRef, svc) { - healthChecks = append(healthChecks, &p) + return addHealthCheckPolicyIndexer(context.Background(), mgr) +} + +func addHealthCheckPolicyIndexer(ctx context.Context, mgr manager.Manager) error { + if err := mgr.GetFieldIndexer().IndexField(ctx, &gwpav1alpha1.HealthCheckPolicy{}, constants.ServicePolicyAttachmentIndex, func(obj client.Object) []string { + policy := obj.(*gwpav1alpha1.HealthCheckPolicy) + targetRef := policy.Spec.TargetRef + var targets []string + if targetRef.Kind == constants.KubernetesServiceKind { + targets = append(targets, types.NamespacedName{ + Namespace: gwutils.NamespaceDerefOr(targetRef.Namespace, policy.Namespace), + Name: string(targetRef.Name), + }.String()) } + + return targets + }); err != nil { + return err } - return healthChecks, nil + return nil +} + +func (r *healthCheckPolicyReconciler) getAttachedHealthChecks(svc client.Object) ([]client.Object, *metav1.Condition) { + c := r.fctx.Manager.GetCache() + key := client.ObjectKeyFromObject(svc).String() + selector := fields.OneTermEqualSelector(constants.ServicePolicyAttachmentIndex, key) + + return gwutils.GetHealthChecks(c, selector), nil } func (r *healthCheckPolicyReconciler) findConflict(healthCheckPolicy client.Object, allHealthCheckPolicies []client.Object, port int32) *types.NamespacedName { @@ -168,13 +172,17 @@ func (r *healthCheckPolicyReconciler) referenceGrantToPolicyAttachment(_ context return nil } - requests := make([]reconcile.Request, 0) - policies := r.fctx.InformerCollection.GetGatewayResourcesFromCache(informers.HealthCheckPoliciesResourceType, false) - - for _, p := range policies { - policy := p.(*gwpav1alpha1.HealthCheckPolicy) + c := r.fctx.Manager.GetCache() + list := &gwpav1alpha1.HealthCheckPolicyList{} + if err := c.List(context.Background(), list); err != nil { + log.Error().Msgf("Failed to list HealthCheckPolicyList: %v", err) + return nil + } + policies := gwutils.ToSlicePtr(list.Items) - if gwutils.HasAccessToTargetRef(policy, policy.Spec.TargetRef, []client.Object{refGrant}) { + requests := make([]reconcile.Request, 0) + for _, policy := range policies { + if gwutils.HasAccessToTargetRef(policy, policy.Spec.TargetRef, []*gwv1beta1.ReferenceGrant{refGrant}) { requests = append(requests, reconcile.Request{ NamespacedName: types.NamespacedName{ Name: policy.Name, diff --git a/pkg/controllers/policyattachment/v1alpha1/loadbalancerpolicy_controller.go b/pkg/controllers/policyattachment/v1alpha1/loadbalancerpolicy_controller.go index c87a15988..d596a80f7 100644 --- a/pkg/controllers/policyattachment/v1alpha1/loadbalancerpolicy_controller.go +++ b/pkg/controllers/policyattachment/v1alpha1/loadbalancerpolicy_controller.go @@ -2,14 +2,16 @@ package v1alpha1 import ( "context" - "fmt" - "sigs.k8s.io/controller-runtime/pkg/handler" - gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" + "k8s.io/apimachinery/pkg/fields" + "sigs.k8s.io/controller-runtime/pkg/manager" + + policystatus "github.com/flomesh-io/fsm/pkg/gateway/status/policy" - "github.com/flomesh-io/fsm/pkg/k8s/informers" + "github.com/flomesh-io/fsm/pkg/constants" - "github.com/flomesh-io/fsm/pkg/gateway/policy/status" + "sigs.k8s.io/controller-runtime/pkg/handler" + gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" "github.com/flomesh-io/fsm/pkg/gateway/policy/utils/loadbalancer" @@ -19,12 +21,6 @@ import ( "k8s.io/apimachinery/pkg/types" - corev1 "k8s.io/api/core/v1" - - metautil "k8s.io/apimachinery/pkg/api/meta" - - gwclient "sigs.k8s.io/gateway-api/pkg/client/clientset/versioned" - "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/record" @@ -35,16 +31,12 @@ import ( fctx "github.com/flomesh-io/fsm/pkg/context" "github.com/flomesh-io/fsm/pkg/controllers" - - policyAttachmentApiClientset "github.com/flomesh-io/fsm/pkg/gen/client/policyattachment/clientset/versioned" ) type loadBalancerPolicyReconciler struct { - recorder record.EventRecorder - fctx *fctx.ControllerContext - gatewayAPIClient gwclient.Interface - policyAttachmentAPIClient policyAttachmentApiClientset.Interface - statusProcessor *status.ServicePolicyStatusProcessor + recorder record.EventRecorder + fctx *fctx.ControllerContext + statusProcessor *policystatus.ServicePolicyStatusProcessor } func (r *loadBalancerPolicyReconciler) NeedLeaderElection() bool { @@ -54,13 +46,11 @@ func (r *loadBalancerPolicyReconciler) NeedLeaderElection() bool { // NewLoadBalancerPolicyReconciler returns a new LoadBalancerPolicy Reconciler func NewLoadBalancerPolicyReconciler(ctx *fctx.ControllerContext) controllers.Reconciler { r := &loadBalancerPolicyReconciler{ - recorder: ctx.Manager.GetEventRecorderFor("LoadBalancerPolicy"), - fctx: ctx, - gatewayAPIClient: gwclient.NewForConfigOrDie(ctx.KubeConfig), - policyAttachmentAPIClient: policyAttachmentApiClientset.NewForConfigOrDie(ctx.KubeConfig), + recorder: ctx.Manager.GetEventRecorderFor("LoadBalancerPolicy"), + fctx: ctx, } - r.statusProcessor = &status.ServicePolicyStatusProcessor{ + r.statusProcessor = &policystatus.ServicePolicyStatusProcessor{ Client: r.fctx.Client, Informer: r.fctx.InformerCollection, GetAttachedPolicies: r.getAttachedLoadBalancers, @@ -88,13 +78,13 @@ func (r *loadBalancerPolicyReconciler) Reconcile(ctx context.Context, req ctrl.R return ctrl.Result{}, nil } - metautil.SetStatusCondition( - &policy.Status.Conditions, - r.statusProcessor.Process(ctx, policy, policy.Spec.TargetRef), - ) - if err := r.fctx.Status().Update(ctx, policy); err != nil { - return ctrl.Result{}, err - } + r.statusProcessor.Process(ctx, r.fctx.StatusUpdater, policystatus.NewPolicyUpdate( + policy, + &policy.ObjectMeta, + &policy.TypeMeta, + policy.Spec.TargetRef, + policy.Status.Conditions, + )) r.fctx.GatewayEventHandler.OnAdd(policy, false) @@ -103,33 +93,45 @@ func (r *loadBalancerPolicyReconciler) Reconcile(ctx context.Context, req ctrl.R // SetupWithManager sets up the controller with the Manager. func (r *loadBalancerPolicyReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). + if err := ctrl.NewControllerManagedBy(mgr). For(&gwpav1alpha1.LoadBalancerPolicy{}). Watches( &gwv1beta1.ReferenceGrant{}, handler.EnqueueRequestsFromMapFunc(r.referenceGrantToPolicyAttachment), ). - Complete(r) -} - -func (r *loadBalancerPolicyReconciler) getAttachedLoadBalancers(policy client.Object, svc client.Object) ([]client.Object, *metav1.Condition) { - loadBalancerPolicyList, err := r.policyAttachmentAPIClient.GatewayV1alpha1().LoadBalancerPolicies(corev1.NamespaceAll).List(context.TODO(), metav1.ListOptions{}) - if err != nil { - return nil, status.ConditionPointer(status.InvalidCondition(policy, fmt.Sprintf("Failed to list LoadBalancerPolicies: %s", err))) + Complete(r); err != nil { + return err } - loadBalancers := make([]client.Object, 0) - referenceGrants := r.fctx.InformerCollection.GetGatewayResourcesFromCache(informers.ReferenceGrantResourceType, false) + return addLoadBalancerPolicyIndexer(context.Background(), mgr) +} - for _, p := range loadBalancerPolicyList.Items { - p := p - if gwutils.IsAcceptedPolicyAttachment(p.Status.Conditions) && - gwutils.IsRefToTarget(referenceGrants, &p, p.Spec.TargetRef, svc) { - loadBalancers = append(loadBalancers, &p) +func addLoadBalancerPolicyIndexer(ctx context.Context, mgr manager.Manager) error { + if err := mgr.GetFieldIndexer().IndexField(ctx, &gwpav1alpha1.LoadBalancerPolicy{}, constants.ServicePolicyAttachmentIndex, func(obj client.Object) []string { + policy := obj.(*gwpav1alpha1.LoadBalancerPolicy) + targetRef := policy.Spec.TargetRef + var targets []string + if targetRef.Kind == constants.KubernetesServiceKind { + targets = append(targets, types.NamespacedName{ + Namespace: gwutils.NamespaceDerefOr(targetRef.Namespace, policy.Namespace), + Name: string(targetRef.Name), + }.String()) } + + return targets + }); err != nil { + return err } - return loadBalancers, nil + return nil +} + +func (r *loadBalancerPolicyReconciler) getAttachedLoadBalancers(svc client.Object) ([]client.Object, *metav1.Condition) { + c := r.fctx.Manager.GetCache() + key := client.ObjectKeyFromObject(svc).String() + selector := fields.OneTermEqualSelector(constants.ServicePolicyAttachmentIndex, key) + + return gwutils.GetLoadBalancers(c, selector), nil } func (r *loadBalancerPolicyReconciler) findConflict(loadBalancerPolicy client.Object, allSessionStickyPolicies []client.Object, port int32) *types.NamespacedName { @@ -168,13 +170,17 @@ func (r *loadBalancerPolicyReconciler) referenceGrantToPolicyAttachment(_ contex return nil } - requests := make([]reconcile.Request, 0) - policies := r.fctx.InformerCollection.GetGatewayResourcesFromCache(informers.LoadBalancerPoliciesResourceType, false) - - for _, p := range policies { - policy := p.(*gwpav1alpha1.LoadBalancerPolicy) + c := r.fctx.Manager.GetCache() + list := &gwpav1alpha1.LoadBalancerPolicyList{} + if err := c.List(context.Background(), list); err != nil { + log.Error().Msgf("Failed to list LoadBalancerPolicyList: %v", err) + return nil + } + policies := gwutils.ToSlicePtr(list.Items) - if gwutils.HasAccessToTargetRef(policy, policy.Spec.TargetRef, []client.Object{refGrant}) { + requests := make([]reconcile.Request, 0) + for _, policy := range policies { + if gwutils.HasAccessToTargetRef(policy, policy.Spec.TargetRef, []*gwv1beta1.ReferenceGrant{refGrant}) { requests = append(requests, reconcile.Request{ NamespacedName: types.NamespacedName{ Name: policy.Name, diff --git a/pkg/controllers/policyattachment/v1alpha1/ratelimitpolicy_controller.go b/pkg/controllers/policyattachment/v1alpha1/ratelimitpolicy_controller.go index ecbbc7ff9..bf21b71c7 100644 --- a/pkg/controllers/policyattachment/v1alpha1/ratelimitpolicy_controller.go +++ b/pkg/controllers/policyattachment/v1alpha1/ratelimitpolicy_controller.go @@ -29,12 +29,18 @@ import ( "fmt" "reflect" - "sigs.k8s.io/controller-runtime/pkg/handler" - gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" + policystatus "github.com/flomesh-io/fsm/pkg/gateway/status/policy" + + "k8s.io/apimachinery/pkg/fields" + + "github.com/flomesh-io/fsm/pkg/gateway/status" - "github.com/flomesh-io/fsm/pkg/k8s/informers" + "sigs.k8s.io/controller-runtime/pkg/cache" - "github.com/flomesh-io/fsm/pkg/gateway/policy/status" + "sigs.k8s.io/controller-runtime/pkg/manager" + + "sigs.k8s.io/controller-runtime/pkg/handler" + gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" gwpkg "github.com/flomesh-io/fsm/pkg/gateway/types" @@ -42,9 +48,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" - corev1 "k8s.io/api/core/v1" - - gwtypes "github.com/flomesh-io/fsm/pkg/gateway/types" gwutils "github.com/flomesh-io/fsm/pkg/gateway/utils" metautil "k8s.io/apimachinery/pkg/api/meta" @@ -52,8 +55,6 @@ import ( "k8s.io/apimachinery/pkg/types" gwv1 "sigs.k8s.io/gateway-api/apis/v1" - gwclient "sigs.k8s.io/gateway-api/pkg/client/clientset/versioned" - gwpav1alpha1 "github.com/flomesh-io/fsm/pkg/apis/policyattachment/v1alpha1" "github.com/flomesh-io/fsm/pkg/constants" @@ -65,16 +66,12 @@ import ( fctx "github.com/flomesh-io/fsm/pkg/context" "github.com/flomesh-io/fsm/pkg/controllers" - - policyAttachmentApiClientset "github.com/flomesh-io/fsm/pkg/gen/client/policyattachment/clientset/versioned" ) type rateLimitPolicyReconciler struct { - recorder record.EventRecorder - fctx *fctx.ControllerContext - gatewayAPIClient gwclient.Interface - policyAttachmentAPIClient policyAttachmentApiClientset.Interface - statusProcessor *status.PolicyStatusProcessor + recorder record.EventRecorder + fctx *fctx.ControllerContext + statusProcessor *policystatus.PolicyStatusProcessor } func (r *rateLimitPolicyReconciler) NeedLeaderElection() bool { @@ -84,13 +81,11 @@ func (r *rateLimitPolicyReconciler) NeedLeaderElection() bool { // NewRateLimitPolicyReconciler returns a new RateLimitPolicy Reconciler func NewRateLimitPolicyReconciler(ctx *fctx.ControllerContext) controllers.Reconciler { r := &rateLimitPolicyReconciler{ - recorder: ctx.Manager.GetEventRecorderFor("RateLimitPolicy"), - fctx: ctx, - gatewayAPIClient: gwclient.NewForConfigOrDie(ctx.KubeConfig), - policyAttachmentAPIClient: policyAttachmentApiClientset.NewForConfigOrDie(ctx.KubeConfig), + recorder: ctx.Manager.GetEventRecorderFor("RateLimitPolicy"), + fctx: ctx, } - r.statusProcessor = &status.PolicyStatusProcessor{ + r.statusProcessor = &policystatus.PolicyStatusProcessor{ Client: r.fctx.Client, Informer: r.fctx.InformerCollection, GetPolicies: r.getRateLimitPolices, @@ -121,79 +116,79 @@ func (r *rateLimitPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Requ return ctrl.Result{}, nil } - metautil.SetStatusCondition( - &policy.Status.Conditions, - r.statusProcessor.Process(ctx, policy, policy.Spec.TargetRef), - ) - if err := r.fctx.Status().Update(ctx, policy); err != nil { - return ctrl.Result{}, err - } + r.statusProcessor.Process(ctx, r.fctx.StatusUpdater, policystatus.NewPolicyUpdate( + policy, + &policy.ObjectMeta, + &policy.TypeMeta, + policy.Spec.TargetRef, + policy.Status.Conditions, + )) r.fctx.GatewayEventHandler.OnAdd(policy, false) return ctrl.Result{}, nil } -func (r *rateLimitPolicyReconciler) getRateLimitPolices(policy client.Object, target client.Object) (map[gwpkg.PolicyMatchType][]client.Object, *metav1.Condition) { - rateLimitPolicyList, err := r.policyAttachmentAPIClient.GatewayV1alpha1().RateLimitPolicies(corev1.NamespaceAll).List(context.TODO(), metav1.ListOptions{}) - if err != nil { - return nil, status.ConditionPointer(status.InvalidCondition(policy, fmt.Sprintf("Failed to list rate limit policies: %s", err))) - } - +func (r *rateLimitPolicyReconciler) getRateLimitPolices(target client.Object) map[gwpkg.PolicyMatchType][]client.Object { + c := r.fctx.Manager.GetCache() policies := make(map[gwpkg.PolicyMatchType][]client.Object) - referenceGrants := r.fctx.InformerCollection.GetGatewayResourcesFromCache(informers.ReferenceGrantResourceType, false) - - for _, p := range rateLimitPolicyList.Items { - p := p - if gwutils.IsAcceptedPolicyAttachment(p.Status.Conditions) { - spec := p.Spec - targetRef := spec.TargetRef - - switch { - case gwutils.IsTargetRefToGVK(targetRef, constants.GatewayGVK) && - gwutils.IsRefToTarget(referenceGrants, &p, targetRef, target) && - len(spec.Ports) > 0: - policies[gwpkg.PolicyMatchTypePort] = append(policies[gwpkg.PolicyMatchTypePort], &p) - case (gwutils.IsTargetRefToGVK(targetRef, constants.HTTPRouteGVK) || gwutils.IsTargetRefToGVK(targetRef, constants.GRPCRouteGVK)) && - gwutils.IsRefToTarget(referenceGrants, &p, targetRef, target) && - len(spec.Hostnames) > 0: - policies[gwpkg.PolicyMatchTypeHostnames] = append(policies[gwpkg.PolicyMatchTypeHostnames], &p) - case gwutils.IsTargetRefToGVK(targetRef, constants.HTTPRouteGVK) && - gwutils.IsRefToTarget(referenceGrants, &p, targetRef, target) && - len(spec.HTTPRateLimits) > 0: - policies[gwpkg.PolicyMatchTypeHTTPRoute] = append(policies[gwpkg.PolicyMatchTypeHTTPRoute], &p) - case gwutils.IsTargetRefToGVK(targetRef, constants.GRPCRouteGVK) && - gwutils.IsRefToTarget(referenceGrants, &p, targetRef, target) && - len(spec.GRPCRateLimits) > 0: - policies[gwpkg.PolicyMatchTypeGRPCRoute] = append(policies[gwpkg.PolicyMatchTypeGRPCRoute], &p) - } + + for _, p := range []struct { + matchType gwpkg.PolicyMatchType + fn func(cache.Cache, fields.Selector) []client.Object + selector fields.Selector + }{ + { + matchType: gwpkg.PolicyMatchTypePort, + fn: gwutils.GetRateLimitsMatchTypePort, + selector: fields.OneTermEqualSelector(constants.PortPolicyAttachmentIndex, client.ObjectKeyFromObject(target).String()), + }, + { + matchType: gwpkg.PolicyMatchTypeHostnames, + fn: gwutils.GetRateLimitsMatchTypeHostname, + selector: fields.OneTermEqualSelector(constants.HostnamePolicyAttachmentIndex, fmt.Sprintf("%s/%s/%s", target.GetObjectKind().GroupVersionKind().Kind, target.GetNamespace(), target.GetName())), + }, + { + matchType: gwpkg.PolicyMatchTypeHTTPRoute, + fn: gwutils.GetRateLimitsMatchTypeHTTPRoute, + selector: fields.OneTermEqualSelector(constants.HTTPRoutePolicyAttachmentIndex, client.ObjectKeyFromObject(target).String()), + }, + { + matchType: gwpkg.PolicyMatchTypeGRPCRoute, + fn: gwutils.GetRateLimitsMatchTypeGRPCRoute, + selector: fields.OneTermEqualSelector(constants.GRPCRoutePolicyAttachmentIndex, client.ObjectKeyFromObject(target).String()), + }, + } { + if result := p.fn(c, p.selector); len(result) > 0 { + policies[p.matchType] = result } } - return policies, nil + return policies } -func (r *rateLimitPolicyReconciler) getConflictedHostnamesBasedRateLimitPolicy(route *gwtypes.RouteContext, rateLimitPolicy client.Object, hostnamesRateLimits []client.Object) *types.NamespacedName { +func (r *rateLimitPolicyReconciler) getConflictedHostnamesBasedRateLimitPolicy(route status.RouteStatusObject, parentRefs []gwv1.ParentReference, rateLimitPolicy client.Object, hostnamesRateLimits []client.Object) *types.NamespacedName { currentPolicy := rateLimitPolicy.(*gwpav1alpha1.RateLimitPolicy) if len(currentPolicy.Spec.Hostnames) == 0 { return nil } - for _, parent := range route.ParentStatus { - if metautil.IsStatusConditionTrue(parent.Conditions, string(gwv1.RouteConditionAccepted)) { - key := getRouteParentKey(route.Meta, parent) + for _, parentRef := range parentRefs { + h := route.StatusUpdateFor(parentRef) + + if metautil.IsStatusConditionTrue(h.ConditionsForParentRef(parentRef), string(gwv1.RouteConditionAccepted)) { + key := getRouteParentKey(route.GetObjectMeta(), parentRef) gateway := &gwv1.Gateway{} if err := r.fctx.Get(context.TODO(), key, gateway); err != nil { continue } - validListeners := gwutils.GetValidListenersForGateway(gateway) + allowedListeners := gwutils.GetAllowedListeners(r.fctx.Manager.GetCache(), gateway, h) - allowedListeners, _ := gwutils.GetAllowedListeners(r.fctx.InformerCollection.GetListers().Namespace, gateway, parent.ParentRef, route, validListeners) for _, listener := range allowedListeners { - hostnames := gwutils.GetValidHostnames(listener.Hostname, route.Hostnames) + hostnames := gwutils.GetValidHostnames(listener.Hostname, route.GetHostnames()) if len(hostnames) == 0 { // no valid hostnames, should ignore it continue @@ -202,12 +197,12 @@ func (r *rateLimitPolicyReconciler) getConflictedHostnamesBasedRateLimitPolicy(r for _, hr := range hostnamesRateLimits { hr := hr.(*gwpav1alpha1.RateLimitPolicy) - r1 := ratelimit.GetRateLimitIfRouteHostnameMatchesPolicy(hostname, *hr) + r1 := ratelimit.GetRateLimitIfRouteHostnameMatchesPolicy(hostname, hr) if r1 == nil { continue } - r2 := ratelimit.GetRateLimitIfRouteHostnameMatchesPolicy(hostname, *currentPolicy) + r2 := ratelimit.GetRateLimitIfRouteHostnameMatchesPolicy(hostname, currentPolicy) if r2 == nil { continue } @@ -245,12 +240,12 @@ func (r *rateLimitPolicyReconciler) getConflictedHTTPRouteBasedRateLimitPolicy(r continue } - r1 := ratelimit.GetRateLimitIfHTTPRouteMatchesPolicy(m, *routePolicy) + r1 := ratelimit.GetRateLimitIfHTTPRouteMatchesPolicy(m, routePolicy) if r1 == nil { continue } - r2 := ratelimit.GetRateLimitIfHTTPRouteMatchesPolicy(m, *currentPolicy) + r2 := ratelimit.GetRateLimitIfHTTPRouteMatchesPolicy(m, currentPolicy) if r2 == nil { continue } @@ -286,12 +281,12 @@ func (r *rateLimitPolicyReconciler) getConflictedGRPCRouteBasedRateLimitPolicy(r continue } - r1 := ratelimit.GetRateLimitIfGRPCRouteMatchesPolicy(m, *routePolicy) + r1 := ratelimit.GetRateLimitIfGRPCRouteMatchesPolicy(m, routePolicy) if r1 == nil { continue } - r2 := ratelimit.GetRateLimitIfGRPCRouteMatchesPolicy(m, *currentPolicy) + r2 := ratelimit.GetRateLimitIfGRPCRouteMatchesPolicy(m, currentPolicy) if r2 == nil { continue } @@ -324,12 +319,12 @@ func (r *rateLimitPolicyReconciler) getConflictedPort(gateway *gwv1.Gateway, rat if len(pr.Spec.Ports) > 0 { for _, listener := range validListeners { - r1 := ratelimit.GetRateLimitIfPortMatchesPolicy(listener.Port, *pr) + r1 := ratelimit.GetRateLimitIfPortMatchesPolicy(listener.Port, pr) if r1 == nil { continue } - r2 := ratelimit.GetRateLimitIfPortMatchesPolicy(listener.Port, *currentPolicy) + r2 := ratelimit.GetRateLimitIfPortMatchesPolicy(listener.Port, currentPolicy) if r2 == nil { continue } @@ -351,13 +346,94 @@ func (r *rateLimitPolicyReconciler) getConflictedPort(gateway *gwv1.Gateway, rat // SetupWithManager sets up the controller with the Manager. func (r *rateLimitPolicyReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). + if err := ctrl.NewControllerManagedBy(mgr). For(&gwpav1alpha1.RateLimitPolicy{}). Watches( &gwv1beta1.ReferenceGrant{}, handler.EnqueueRequestsFromMapFunc(r.referenceGrantToPolicyAttachment), ). - Complete(r) + Complete(r); err != nil { + return err + } + + return addRateLimitPolicyIndexer(context.Background(), mgr) +} + +func addRateLimitPolicyIndexer(ctx context.Context, mgr manager.Manager) error { + if err := mgr.GetFieldIndexer().IndexField(ctx, &gwpav1alpha1.RateLimitPolicy{}, constants.PortPolicyAttachmentIndex, addRateLimitPortIndexFunc); err != nil { + return err + } + + if err := mgr.GetFieldIndexer().IndexField(ctx, &gwpav1alpha1.RateLimitPolicy{}, constants.HostnamePolicyAttachmentIndex, addRateLimitHostnameIndexFunc); err != nil { + return err + } + + if err := mgr.GetFieldIndexer().IndexField(ctx, &gwpav1alpha1.RateLimitPolicy{}, constants.HTTPRoutePolicyAttachmentIndex, addRateLimitHTTPRouteIndexFunc); err != nil { + return err + } + + if err := mgr.GetFieldIndexer().IndexField(ctx, &gwpav1alpha1.RateLimitPolicy{}, constants.GRPCRoutePolicyAttachmentIndex, addRateLimitGRPCRouteIndexFunc); err != nil { + return err + } + + return nil +} + +func addRateLimitPortIndexFunc(obj client.Object) []string { + policy := obj.(*gwpav1alpha1.RateLimitPolicy) + targetRef := policy.Spec.TargetRef + + var targets []string + if gwutils.IsTargetRefToGVK(targetRef, constants.GatewayGVK) && len(policy.Spec.Ports) > 0 { + targets = append(targets, types.NamespacedName{ + Namespace: gwutils.NamespaceDerefOr(targetRef.Namespace, policy.Namespace), + Name: string(targetRef.Name), + }.String()) + } + + return targets +} + +func addRateLimitHostnameIndexFunc(obj client.Object) []string { + policy := obj.(*gwpav1alpha1.RateLimitPolicy) + targetRef := policy.Spec.TargetRef + + var targets []string + if (gwutils.IsTargetRefToGVK(targetRef, constants.HTTPRouteGVK) || gwutils.IsTargetRefToGVK(targetRef, constants.GRPCRouteGVK)) && len(policy.Spec.Hostnames) > 0 { + targets = append(targets, fmt.Sprintf("%s/%s/%s", targetRef.Kind, gwutils.NamespaceDerefOr(targetRef.Namespace, policy.Namespace), string(targetRef.Name))) + } + + return targets +} + +func addRateLimitHTTPRouteIndexFunc(obj client.Object) []string { + policy := obj.(*gwpav1alpha1.RateLimitPolicy) + targetRef := policy.Spec.TargetRef + + var targets []string + if gwutils.IsTargetRefToGVK(targetRef, constants.HTTPRouteGVK) && len(policy.Spec.HTTPRateLimits) > 0 { + targets = append(targets, types.NamespacedName{ + Namespace: gwutils.NamespaceDerefOr(targetRef.Namespace, policy.Namespace), + Name: string(targetRef.Name), + }.String()) + } + + return targets +} + +func addRateLimitGRPCRouteIndexFunc(obj client.Object) []string { + policy := obj.(*gwpav1alpha1.RateLimitPolicy) + targetRef := policy.Spec.TargetRef + + var targets []string + if gwutils.IsTargetRefToGVK(targetRef, constants.GRPCRouteGVK) && len(policy.Spec.GRPCRateLimits) > 0 { + targets = append(targets, types.NamespacedName{ + Namespace: gwutils.NamespaceDerefOr(targetRef.Namespace, policy.Namespace), + Name: string(targetRef.Name), + }.String()) + } + + return targets } func (r *rateLimitPolicyReconciler) referenceGrantToPolicyAttachment(_ context.Context, obj client.Object) []reconcile.Request { @@ -367,13 +443,17 @@ func (r *rateLimitPolicyReconciler) referenceGrantToPolicyAttachment(_ context.C return nil } - requests := make([]reconcile.Request, 0) - policies := r.fctx.InformerCollection.GetGatewayResourcesFromCache(informers.RateLimitPoliciesResourceType, false) - - for _, p := range policies { - policy := p.(*gwpav1alpha1.RateLimitPolicy) + c := r.fctx.Manager.GetCache() + list := &gwpav1alpha1.RateLimitPolicyList{} + if err := c.List(context.Background(), list); err != nil { + log.Error().Msgf("Failed to list RateLimitPolicyList: %v", err) + return nil + } + policies := gwutils.ToSlicePtr(list.Items) - if gwutils.HasAccessToTargetRef(policy, policy.Spec.TargetRef, []client.Object{refGrant}) { + requests := make([]reconcile.Request, 0) + for _, policy := range policies { + if gwutils.HasAccessToTargetRef(policy, policy.Spec.TargetRef, []*gwv1beta1.ReferenceGrant{refGrant}) { requests = append(requests, reconcile.Request{ NamespacedName: types.NamespacedName{ Name: policy.Name, diff --git a/pkg/controllers/policyattachment/v1alpha1/retrypolicy_controller.go b/pkg/controllers/policyattachment/v1alpha1/retrypolicy_controller.go index 3a442e06e..50db13749 100644 --- a/pkg/controllers/policyattachment/v1alpha1/retrypolicy_controller.go +++ b/pkg/controllers/policyattachment/v1alpha1/retrypolicy_controller.go @@ -2,15 +2,17 @@ package v1alpha1 import ( "context" - "fmt" "reflect" - "sigs.k8s.io/controller-runtime/pkg/handler" - gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" + policystatus "github.com/flomesh-io/fsm/pkg/gateway/status/policy" + + "k8s.io/apimachinery/pkg/fields" + "sigs.k8s.io/controller-runtime/pkg/manager" - "github.com/flomesh-io/fsm/pkg/k8s/informers" + "github.com/flomesh-io/fsm/pkg/constants" - "github.com/flomesh-io/fsm/pkg/gateway/policy/status" + "sigs.k8s.io/controller-runtime/pkg/handler" + gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" "github.com/flomesh-io/fsm/pkg/gateway/policy/utils/retry" @@ -20,12 +22,6 @@ import ( "k8s.io/apimachinery/pkg/types" - corev1 "k8s.io/api/core/v1" - - metautil "k8s.io/apimachinery/pkg/api/meta" - - gwclient "sigs.k8s.io/gateway-api/pkg/client/clientset/versioned" - "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/record" @@ -36,16 +32,12 @@ import ( fctx "github.com/flomesh-io/fsm/pkg/context" "github.com/flomesh-io/fsm/pkg/controllers" - - policyAttachmentApiClientset "github.com/flomesh-io/fsm/pkg/gen/client/policyattachment/clientset/versioned" ) type retryPolicyReconciler struct { - recorder record.EventRecorder - fctx *fctx.ControllerContext - gatewayAPIClient gwclient.Interface - policyAttachmentAPIClient policyAttachmentApiClientset.Interface - statusProcessor *status.ServicePolicyStatusProcessor + recorder record.EventRecorder + fctx *fctx.ControllerContext + statusProcessor *policystatus.ServicePolicyStatusProcessor } func (r *retryPolicyReconciler) NeedLeaderElection() bool { @@ -55,13 +47,11 @@ func (r *retryPolicyReconciler) NeedLeaderElection() bool { // NewRetryPolicyReconciler returns a new RetryPolicy Reconciler func NewRetryPolicyReconciler(ctx *fctx.ControllerContext) controllers.Reconciler { r := &retryPolicyReconciler{ - recorder: ctx.Manager.GetEventRecorderFor("RetryPolicy"), - fctx: ctx, - gatewayAPIClient: gwclient.NewForConfigOrDie(ctx.KubeConfig), - policyAttachmentAPIClient: policyAttachmentApiClientset.NewForConfigOrDie(ctx.KubeConfig), + recorder: ctx.Manager.GetEventRecorderFor("RetryPolicy"), + fctx: ctx, } - r.statusProcessor = &status.ServicePolicyStatusProcessor{ + r.statusProcessor = &policystatus.ServicePolicyStatusProcessor{ Client: r.fctx.Client, Informer: r.fctx.InformerCollection, GetAttachedPolicies: r.getAttachedRetryPolicies, @@ -89,13 +79,13 @@ func (r *retryPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, nil } - metautil.SetStatusCondition( - &policy.Status.Conditions, - r.statusProcessor.Process(ctx, policy, policy.Spec.TargetRef), - ) - if err := r.fctx.Status().Update(ctx, policy); err != nil { - return ctrl.Result{}, err - } + r.statusProcessor.Process(ctx, r.fctx.StatusUpdater, policystatus.NewPolicyUpdate( + policy, + &policy.ObjectMeta, + &policy.TypeMeta, + policy.Spec.TargetRef, + policy.Status.Conditions, + )) r.fctx.GatewayEventHandler.OnAdd(policy, false) @@ -104,33 +94,45 @@ func (r *retryPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Request) // SetupWithManager sets up the controller with the Manager. func (r *retryPolicyReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). + if err := ctrl.NewControllerManagedBy(mgr). For(&gwpav1alpha1.RetryPolicy{}). Watches( &gwv1beta1.ReferenceGrant{}, handler.EnqueueRequestsFromMapFunc(r.referenceGrantToPolicyAttachment), ). - Complete(r) -} - -func (r *retryPolicyReconciler) getAttachedRetryPolicies(policy client.Object, svc client.Object) ([]client.Object, *metav1.Condition) { - retryPolicyList, err := r.policyAttachmentAPIClient.GatewayV1alpha1().RetryPolicies(corev1.NamespaceAll).List(context.TODO(), metav1.ListOptions{}) - if err != nil { - return nil, status.ConditionPointer(status.InvalidCondition(policy, fmt.Sprintf("Failed to list RetryPolicies: %s", err))) + Complete(r); err != nil { + return err } - retryPolicies := make([]client.Object, 0) - referenceGrants := r.fctx.InformerCollection.GetGatewayResourcesFromCache(informers.ReferenceGrantResourceType, false) + return addRetryPolicyIndexer(context.Background(), mgr) +} - for _, p := range retryPolicyList.Items { - p := p - if gwutils.IsAcceptedPolicyAttachment(p.Status.Conditions) && - gwutils.IsRefToTarget(referenceGrants, &p, p.Spec.TargetRef, svc) { - retryPolicies = append(retryPolicies, &p) +func addRetryPolicyIndexer(ctx context.Context, mgr manager.Manager) error { + if err := mgr.GetFieldIndexer().IndexField(ctx, &gwpav1alpha1.RetryPolicy{}, constants.ServicePolicyAttachmentIndex, func(obj client.Object) []string { + policy := obj.(*gwpav1alpha1.RetryPolicy) + targetRef := policy.Spec.TargetRef + var targets []string + if targetRef.Kind == constants.KubernetesServiceKind { + targets = append(targets, types.NamespacedName{ + Namespace: gwutils.NamespaceDerefOr(targetRef.Namespace, policy.Namespace), + Name: string(targetRef.Name), + }.String()) } + + return targets + }); err != nil { + return err } - return retryPolicies, nil + return nil +} + +func (r *retryPolicyReconciler) getAttachedRetryPolicies(svc client.Object) ([]client.Object, *metav1.Condition) { + c := r.fctx.Manager.GetCache() + key := client.ObjectKeyFromObject(svc).String() + selector := fields.OneTermEqualSelector(constants.ServicePolicyAttachmentIndex, key) + + return gwutils.GetRetries(c, selector), nil } func (r *retryPolicyReconciler) findConflict(retryPolicy client.Object, allRetryPolicies []client.Object, port int32) *types.NamespacedName { @@ -169,13 +171,17 @@ func (r *retryPolicyReconciler) referenceGrantToPolicyAttachment(_ context.Conte return nil } - requests := make([]reconcile.Request, 0) - policies := r.fctx.InformerCollection.GetGatewayResourcesFromCache(informers.RetryPoliciesResourceType, false) - - for _, p := range policies { - policy := p.(*gwpav1alpha1.RetryPolicy) + c := r.fctx.Manager.GetCache() + list := &gwpav1alpha1.RetryPolicyList{} + if err := c.List(context.Background(), list); err != nil { + log.Error().Msgf("Failed to list RetryPolicyList: %v", err) + return nil + } + policies := gwutils.ToSlicePtr(list.Items) - if gwutils.HasAccessToTargetRef(policy, policy.Spec.TargetRef, []client.Object{refGrant}) { + requests := make([]reconcile.Request, 0) + for _, policy := range policies { + if gwutils.HasAccessToTargetRef(policy, policy.Spec.TargetRef, []*gwv1beta1.ReferenceGrant{refGrant}) { requests = append(requests, reconcile.Request{ NamespacedName: types.NamespacedName{ Name: policy.Name, diff --git a/pkg/controllers/policyattachment/v1alpha1/sessionstickypolicy_controller.go b/pkg/controllers/policyattachment/v1alpha1/sessionstickypolicy_controller.go index 68e03a20c..fcaa9ac25 100644 --- a/pkg/controllers/policyattachment/v1alpha1/sessionstickypolicy_controller.go +++ b/pkg/controllers/policyattachment/v1alpha1/sessionstickypolicy_controller.go @@ -2,15 +2,17 @@ package v1alpha1 import ( "context" - "fmt" "reflect" - "sigs.k8s.io/controller-runtime/pkg/handler" - gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" + policystatus "github.com/flomesh-io/fsm/pkg/gateway/status/policy" + + "k8s.io/apimachinery/pkg/fields" + "sigs.k8s.io/controller-runtime/pkg/manager" - "github.com/flomesh-io/fsm/pkg/k8s/informers" + "github.com/flomesh-io/fsm/pkg/constants" - "github.com/flomesh-io/fsm/pkg/gateway/policy/status" + "sigs.k8s.io/controller-runtime/pkg/handler" + gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" "github.com/flomesh-io/fsm/pkg/gateway/policy/utils/sessionsticky" @@ -20,12 +22,6 @@ import ( "k8s.io/apimachinery/pkg/types" - corev1 "k8s.io/api/core/v1" - - metautil "k8s.io/apimachinery/pkg/api/meta" - - gwclient "sigs.k8s.io/gateway-api/pkg/client/clientset/versioned" - "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/record" @@ -36,16 +32,12 @@ import ( fctx "github.com/flomesh-io/fsm/pkg/context" "github.com/flomesh-io/fsm/pkg/controllers" - - policyAttachmentApiClientset "github.com/flomesh-io/fsm/pkg/gen/client/policyattachment/clientset/versioned" ) type sessionStickyPolicyReconciler struct { - recorder record.EventRecorder - fctx *fctx.ControllerContext - gatewayAPIClient gwclient.Interface - policyAttachmentAPIClient policyAttachmentApiClientset.Interface - statusProcessor *status.ServicePolicyStatusProcessor + recorder record.EventRecorder + fctx *fctx.ControllerContext + statusProcessor *policystatus.ServicePolicyStatusProcessor } func (r *sessionStickyPolicyReconciler) NeedLeaderElection() bool { @@ -55,13 +47,11 @@ func (r *sessionStickyPolicyReconciler) NeedLeaderElection() bool { // NewSessionStickyPolicyReconciler returns a new SessionStickyPolicy Reconciler func NewSessionStickyPolicyReconciler(ctx *fctx.ControllerContext) controllers.Reconciler { r := &sessionStickyPolicyReconciler{ - recorder: ctx.Manager.GetEventRecorderFor("SessionStickyPolicy"), - fctx: ctx, - gatewayAPIClient: gwclient.NewForConfigOrDie(ctx.KubeConfig), - policyAttachmentAPIClient: policyAttachmentApiClientset.NewForConfigOrDie(ctx.KubeConfig), + recorder: ctx.Manager.GetEventRecorderFor("SessionStickyPolicy"), + fctx: ctx, } - r.statusProcessor = &status.ServicePolicyStatusProcessor{ + r.statusProcessor = &policystatus.ServicePolicyStatusProcessor{ Client: r.fctx.Client, Informer: r.fctx.InformerCollection, GetAttachedPolicies: r.getAttachedSessionStickies, @@ -89,13 +79,13 @@ func (r *sessionStickyPolicyReconciler) Reconcile(ctx context.Context, req ctrl. return ctrl.Result{}, nil } - metautil.SetStatusCondition( - &policy.Status.Conditions, - r.statusProcessor.Process(ctx, policy, policy.Spec.TargetRef), - ) - if err := r.fctx.Status().Update(ctx, policy); err != nil { - return ctrl.Result{}, err - } + r.statusProcessor.Process(ctx, r.fctx.StatusUpdater, policystatus.NewPolicyUpdate( + policy, + &policy.ObjectMeta, + &policy.TypeMeta, + policy.Spec.TargetRef, + policy.Status.Conditions, + )) r.fctx.GatewayEventHandler.OnAdd(policy, false) @@ -104,33 +94,45 @@ func (r *sessionStickyPolicyReconciler) Reconcile(ctx context.Context, req ctrl. // SetupWithManager sets up the controller with the Manager. func (r *sessionStickyPolicyReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). + if err := ctrl.NewControllerManagedBy(mgr). For(&gwpav1alpha1.SessionStickyPolicy{}). Watches( &gwv1beta1.ReferenceGrant{}, handler.EnqueueRequestsFromMapFunc(r.referenceGrantToPolicyAttachment), ). - Complete(r) -} - -func (r *sessionStickyPolicyReconciler) getAttachedSessionStickies(policy client.Object, svc client.Object) ([]client.Object, *metav1.Condition) { - sessionStickyPolicyList, err := r.policyAttachmentAPIClient.GatewayV1alpha1().SessionStickyPolicies(corev1.NamespaceAll).List(context.TODO(), metav1.ListOptions{}) - if err != nil { - return nil, status.ConditionPointer(status.InvalidCondition(policy, fmt.Sprintf("Failed to list SessionStickyPolicies: %s", err))) + Complete(r); err != nil { + return err } - sessionStickies := make([]client.Object, 0) - referenceGrants := r.fctx.InformerCollection.GetGatewayResourcesFromCache(informers.ReferenceGrantResourceType, false) + return addSessionStickyPolicyIndexer(context.Background(), mgr) +} - for _, p := range sessionStickyPolicyList.Items { - p := p - if gwutils.IsAcceptedPolicyAttachment(p.Status.Conditions) && - gwutils.IsRefToTarget(referenceGrants, &p, p.Spec.TargetRef, svc) { - sessionStickies = append(sessionStickies, &p) +func addSessionStickyPolicyIndexer(ctx context.Context, mgr manager.Manager) error { + if err := mgr.GetFieldIndexer().IndexField(ctx, &gwpav1alpha1.SessionStickyPolicy{}, constants.ServicePolicyAttachmentIndex, func(obj client.Object) []string { + policy := obj.(*gwpav1alpha1.SessionStickyPolicy) + targetRef := policy.Spec.TargetRef + var targets []string + if targetRef.Kind == constants.KubernetesServiceKind { + targets = append(targets, types.NamespacedName{ + Namespace: gwutils.NamespaceDerefOr(targetRef.Namespace, policy.Namespace), + Name: string(targetRef.Name), + }.String()) } + + return targets + }); err != nil { + return err } - return sessionStickies, nil + return nil +} + +func (r *sessionStickyPolicyReconciler) getAttachedSessionStickies(svc client.Object) ([]client.Object, *metav1.Condition) { + c := r.fctx.Manager.GetCache() + key := client.ObjectKeyFromObject(svc).String() + selector := fields.OneTermEqualSelector(constants.ServicePolicyAttachmentIndex, key) + + return gwutils.GetSessionStickies(c, selector), nil } func (r *sessionStickyPolicyReconciler) findConflict(sessionStickyPolicy client.Object, allSessionStickyPolicies []client.Object, port int32) *types.NamespacedName { @@ -169,13 +171,17 @@ func (r *sessionStickyPolicyReconciler) referenceGrantToPolicyAttachment(_ conte return nil } - requests := make([]reconcile.Request, 0) - policies := r.fctx.InformerCollection.GetGatewayResourcesFromCache(informers.SessionStickyPoliciesResourceType, false) - - for _, p := range policies { - policy := p.(*gwpav1alpha1.SessionStickyPolicy) + c := r.fctx.Manager.GetCache() + list := &gwpav1alpha1.SessionStickyPolicyList{} + if err := c.List(context.Background(), list); err != nil { + log.Error().Msgf("Failed to list SessionStickyPolicyList: %v", err) + return nil + } + policies := gwutils.ToSlicePtr(list.Items) - if gwutils.HasAccessToTargetRef(policy, policy.Spec.TargetRef, []client.Object{refGrant}) { + requests := make([]reconcile.Request, 0) + for _, policy := range policies { + if gwutils.HasAccessToTargetRef(policy, policy.Spec.TargetRef, []*gwv1beta1.ReferenceGrant{refGrant}) { requests = append(requests, reconcile.Request{ NamespacedName: types.NamespacedName{ Name: policy.Name, diff --git a/pkg/controllers/policyattachment/v1alpha1/upstreamtlspolicy_controller.go b/pkg/controllers/policyattachment/v1alpha1/upstreamtlspolicy_controller.go index e41a0952c..117abd5cc 100644 --- a/pkg/controllers/policyattachment/v1alpha1/upstreamtlspolicy_controller.go +++ b/pkg/controllers/policyattachment/v1alpha1/upstreamtlspolicy_controller.go @@ -2,15 +2,18 @@ package v1alpha1 import ( "context" - "fmt" "reflect" - "sigs.k8s.io/controller-runtime/pkg/handler" - gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" + policystatus "github.com/flomesh-io/fsm/pkg/gateway/status/policy" + + "k8s.io/apimachinery/pkg/fields" + + "sigs.k8s.io/controller-runtime/pkg/manager" - "github.com/flomesh-io/fsm/pkg/k8s/informers" + "github.com/flomesh-io/fsm/pkg/constants" - "github.com/flomesh-io/fsm/pkg/gateway/policy/status" + "sigs.k8s.io/controller-runtime/pkg/handler" + gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" "github.com/flomesh-io/fsm/pkg/gateway/policy/utils/upstreamtls" @@ -20,12 +23,6 @@ import ( "k8s.io/apimachinery/pkg/types" - corev1 "k8s.io/api/core/v1" - - metautil "k8s.io/apimachinery/pkg/api/meta" - - gwclient "sigs.k8s.io/gateway-api/pkg/client/clientset/versioned" - "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/record" @@ -36,16 +33,12 @@ import ( fctx "github.com/flomesh-io/fsm/pkg/context" "github.com/flomesh-io/fsm/pkg/controllers" - - policyAttachmentApiClientset "github.com/flomesh-io/fsm/pkg/gen/client/policyattachment/clientset/versioned" ) type upstreamTLSPolicyReconciler struct { - recorder record.EventRecorder - fctx *fctx.ControllerContext - gatewayAPIClient gwclient.Interface - policyAttachmentAPIClient policyAttachmentApiClientset.Interface - statusProcessor *status.ServicePolicyStatusProcessor + recorder record.EventRecorder + fctx *fctx.ControllerContext + statusProcessor *policystatus.ServicePolicyStatusProcessor } func (r *upstreamTLSPolicyReconciler) NeedLeaderElection() bool { @@ -55,13 +48,11 @@ func (r *upstreamTLSPolicyReconciler) NeedLeaderElection() bool { // NewUpstreamTLSPolicyReconciler returns a new UpstreamTLSPolicy Reconciler func NewUpstreamTLSPolicyReconciler(ctx *fctx.ControllerContext) controllers.Reconciler { r := &upstreamTLSPolicyReconciler{ - recorder: ctx.Manager.GetEventRecorderFor("UpstreamTLSPolicy"), - fctx: ctx, - gatewayAPIClient: gwclient.NewForConfigOrDie(ctx.KubeConfig), - policyAttachmentAPIClient: policyAttachmentApiClientset.NewForConfigOrDie(ctx.KubeConfig), + recorder: ctx.Manager.GetEventRecorderFor("UpstreamTLSPolicy"), + fctx: ctx, } - r.statusProcessor = &status.ServicePolicyStatusProcessor{ + r.statusProcessor = &policystatus.ServicePolicyStatusProcessor{ Client: r.fctx.Client, Informer: r.fctx.InformerCollection, GetAttachedPolicies: r.getAttachedUpstreamTLSPolices, @@ -89,13 +80,13 @@ func (r *upstreamTLSPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Re return ctrl.Result{}, nil } - metautil.SetStatusCondition( - &policy.Status.Conditions, - r.statusProcessor.Process(ctx, policy, policy.Spec.TargetRef), - ) - if err := r.fctx.Status().Update(ctx, policy); err != nil { - return ctrl.Result{}, err - } + r.statusProcessor.Process(ctx, r.fctx.StatusUpdater, policystatus.NewPolicyUpdate( + policy, + &policy.ObjectMeta, + &policy.TypeMeta, + policy.Spec.TargetRef, + policy.Status.Conditions, + )) r.fctx.GatewayEventHandler.OnAdd(policy, false) @@ -104,33 +95,84 @@ func (r *upstreamTLSPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Re // SetupWithManager sets up the controller with the Manager. func (r *upstreamTLSPolicyReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). + if err := ctrl.NewControllerManagedBy(mgr). For(&gwpav1alpha1.UpstreamTLSPolicy{}). Watches( &gwv1beta1.ReferenceGrant{}, handler.EnqueueRequestsFromMapFunc(r.referenceGrantToPolicyAttachment), ). - Complete(r) + Complete(r); err != nil { + return err + } + + return addUpstreamTLSPolicyIndexer(context.Background(), mgr) } -func (r *upstreamTLSPolicyReconciler) getAttachedUpstreamTLSPolices(policy client.Object, svc client.Object) ([]client.Object, *metav1.Condition) { - upstreamTLSPolicyList, err := r.policyAttachmentAPIClient.GatewayV1alpha1().UpstreamTLSPolicies(corev1.NamespaceAll).List(context.TODO(), metav1.ListOptions{}) - if err != nil { - return nil, status.ConditionPointer(status.InvalidCondition(policy, fmt.Sprintf("Failed to list UpstreamTLSPolicies: %s", err))) +func addUpstreamTLSPolicyIndexer(ctx context.Context, mgr manager.Manager) error { + if err := mgr.GetFieldIndexer().IndexField(ctx, &gwpav1alpha1.UpstreamTLSPolicy{}, constants.ServicePolicyAttachmentIndex, func(obj client.Object) []string { + policy := obj.(*gwpav1alpha1.UpstreamTLSPolicy) + targetRef := policy.Spec.TargetRef + var targets []string + if targetRef.Kind == constants.KubernetesServiceKind { + targets = append(targets, types.NamespacedName{ + Namespace: gwutils.NamespaceDerefOr(targetRef.Namespace, policy.Namespace), + Name: string(targetRef.Name), + }.String()) + } + + return targets + }); err != nil { + return err } - upstreamTLSPolicies := make([]client.Object, 0) - referenceGrants := r.fctx.InformerCollection.GetGatewayResourcesFromCache(informers.ReferenceGrantResourceType, false) + if err := mgr.GetFieldIndexer().IndexField(ctx, &gwpav1alpha1.UpstreamTLSPolicy{}, constants.SecretUpstreamTLSPolicyIndex, addSecretUpstreamTLSPolicyFunc); err != nil { + return err + } + + return nil +} - for _, p := range upstreamTLSPolicyList.Items { - p := p - if gwutils.IsAcceptedPolicyAttachment(p.Status.Conditions) && - gwutils.IsRefToTarget(referenceGrants, &p, p.Spec.TargetRef, svc) { - upstreamTLSPolicies = append(upstreamTLSPolicies, &p) +func addSecretUpstreamTLSPolicyFunc(obj client.Object) []string { + policy := obj.(*gwpav1alpha1.UpstreamTLSPolicy) + secrets := make([]string, 0) + + if policy.Spec.DefaultConfig != nil { + ref := policy.Spec.DefaultConfig.CertificateRef + kind := ref.Kind + if kind == nil || string(*kind) == constants.KubernetesSecretKind { + secrets = append(secrets, types.NamespacedName{ + Namespace: gwutils.NamespaceDerefOr(ref.Namespace, policy.Namespace), + Name: string(ref.Name), + }.String()) } } - return upstreamTLSPolicies, nil + if len(policy.Spec.Ports) > 0 { + for _, port := range policy.Spec.Ports { + if port.Config == nil { + continue + } + + ref := port.Config.CertificateRef + kind := ref.Kind + if kind == nil || string(*kind) == constants.KubernetesSecretKind { + secrets = append(secrets, types.NamespacedName{ + Namespace: gwutils.NamespaceDerefOr(ref.Namespace, policy.Namespace), + Name: string(ref.Name), + }.String()) + } + } + } + + return secrets +} + +func (r *upstreamTLSPolicyReconciler) getAttachedUpstreamTLSPolices(svc client.Object) ([]client.Object, *metav1.Condition) { + c := r.fctx.Manager.GetCache() + key := client.ObjectKeyFromObject(svc).String() + selector := fields.OneTermEqualSelector(constants.ServicePolicyAttachmentIndex, key) + + return gwutils.GetUpStreamTLSes(c, selector), nil } func (r *upstreamTLSPolicyReconciler) findConflict(upstreamTLSPolicy client.Object, allUpstreamTLSPolicies []client.Object, port int32) *types.NamespacedName { @@ -169,13 +211,17 @@ func (r *upstreamTLSPolicyReconciler) referenceGrantToPolicyAttachment(_ context return nil } - requests := make([]reconcile.Request, 0) - policies := r.fctx.InformerCollection.GetGatewayResourcesFromCache(informers.UpstreamTLSPoliciesResourceType, false) - - for _, p := range policies { - policy := p.(*gwpav1alpha1.UpstreamTLSPolicy) + c := r.fctx.Manager.GetCache() + list := &gwpav1alpha1.UpstreamTLSPolicyList{} + if err := c.List(context.Background(), list); err != nil { + log.Error().Msgf("Failed to list UpstreamTLSPolicyList: %v", err) + return nil + } + policies := gwutils.ToSlicePtr(list.Items) - if gwutils.HasAccessToTargetRef(policy, policy.Spec.TargetRef, []client.Object{refGrant}) { + requests := make([]reconcile.Request, 0) + for _, policy := range policies { + if gwutils.HasAccessToTargetRef(policy, policy.Spec.TargetRef, []*gwv1beta1.ReferenceGrant{refGrant}) { requests = append(requests, reconcile.Request{ NamespacedName: types.NamespacedName{ Name: policy.Name, diff --git a/pkg/controllers/policyattachment/v1alpha1/utils.go b/pkg/controllers/policyattachment/v1alpha1/utils.go index 4f8aa12c5..70b9b9af3 100644 --- a/pkg/controllers/policyattachment/v1alpha1/utils.go +++ b/pkg/controllers/policyattachment/v1alpha1/utils.go @@ -8,9 +8,9 @@ import ( gwutils "github.com/flomesh-io/fsm/pkg/gateway/utils" ) -func getRouteParentKey(route metav1.Object, parent gwv1.RouteParentStatus) types.NamespacedName { +func getRouteParentKey(route metav1.Object, parentRef gwv1.ParentReference) types.NamespacedName { return types.NamespacedName{ - Namespace: gwutils.Namespace(parent.ParentRef.Namespace, route.GetNamespace()), - Name: string(parent.ParentRef.Name), + Namespace: gwutils.NamespaceDerefOr(parentRef.Namespace, route.GetNamespace()), + Name: string(parentRef.Name), } } diff --git a/pkg/gateway/cache/cache.go b/pkg/gateway/cache/cache.go index ecca8a376..2c2ce7ec7 100644 --- a/pkg/gateway/cache/cache.go +++ b/pkg/gateway/cache/cache.go @@ -5,6 +5,9 @@ import ( "fmt" "sync" + "sigs.k8s.io/controller-runtime/pkg/cache" + + cctx "github.com/flomesh-io/fsm/pkg/context" "github.com/flomesh-io/fsm/pkg/version" gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" @@ -16,7 +19,6 @@ import ( mcsv1alpha1 "github.com/flomesh-io/fsm/pkg/apis/multicluster/v1alpha1" gwpav1alpha1 "github.com/flomesh-io/fsm/pkg/apis/policyattachment/v1alpha1" - "k8s.io/client-go/kubernetes" gwv1 "sigs.k8s.io/gateway-api/apis/v1" "github.com/flomesh-io/fsm/pkg/configurator" @@ -26,24 +28,23 @@ import ( // GatewayCache is a cache of all the resources that are relevant to the gateway type GatewayCache struct { - repoClient *repo.PipyRepoClient - informers *informers.InformerCollection - kubeClient kubernetes.Interface - cfg configurator.Configurator - triggers map[informers.ResourceType]Trigger - gatewayclass *gwv1.GatewayClass + repoClient *repo.PipyRepoClient + client cache.Cache + cfg configurator.Configurator + triggers map[informers.ResourceType]Trigger + //gatewayclass *gwv1.GatewayClass mutex *sync.RWMutex useEndpointSlices bool } // NewGatewayCache creates a new gateway cache -func NewGatewayCache(informerCollection *informers.InformerCollection, kubeClient kubernetes.Interface, cfg configurator.Configurator) *GatewayCache { +func NewGatewayCache(ctx *cctx.ControllerContext) *GatewayCache { + cfg := ctx.Configurator repoBaseURL := fmt.Sprintf("%s://%s:%d", "http", cfg.GetRepoServerIPAddr(), cfg.GetProxyServerPort()) - useEndpointSlices := cfg.GetFeatureFlags().UseEndpointSlicesForGateway && version.IsEndpointSliceEnabled(kubeClient) + useEndpointSlices := cfg.GetFeatureFlags().UseEndpointSlicesForGateway && version.IsEndpointSliceEnabled(ctx.KubeClient) return &GatewayCache{ repoClient: repo.NewRepoClient(repoBaseURL, cfg.GetFSMLogLevel()), - informers: informerCollection, - kubeClient: kubeClient, + client: ctx.Manager.GetCache(), cfg: cfg, triggers: map[informers.ResourceType]Trigger{ diff --git a/pkg/gateway/cache/config.go b/pkg/gateway/cache/config.go index b74f7b0d6..8d966b162 100644 --- a/pkg/gateway/cache/config.go +++ b/pkg/gateway/cache/config.go @@ -3,9 +3,9 @@ package cache import ( "fmt" - gwv1 "sigs.k8s.io/gateway-api/apis/v1" + gwutils "github.com/flomesh-io/fsm/pkg/gateway/utils" - "github.com/flomesh-io/fsm/pkg/k8s/informers" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" "github.com/tidwall/gjson" @@ -20,7 +20,7 @@ func (c *GatewayCache) BuildConfigs() { defer c.mutex.Unlock() syncConfig := func(gateway *gwv1.Gateway, config *fgw.ConfigSpec) { - gatewayPath := utils.GatewayCodebasePath(gateway.Namespace) + gatewayPath := utils.GatewayCodebasePath(gateway.Namespace, gateway.Name) if exists := c.repoClient.CodebaseExists(gatewayPath); !exists { return } @@ -51,11 +51,8 @@ func (c *GatewayCache) BuildConfigs() { } } - policies := c.policyAttachments() - referenceGrants := c.getResourcesFromCache(informers.ReferenceGrantResourceType, false) - - for _, gw := range c.getActiveGateways() { - cfg := NewGatewayProcessor(c, gw, policies, referenceGrants).build() + for _, gw := range gwutils.GetActiveGateways(c.client) { + cfg := NewGatewayProcessor(c, gw).build() go syncConfig(gw, cfg) } diff --git a/pkg/gateway/cache/gatewayclasses_trigger.go b/pkg/gateway/cache/gatewayclasses_trigger.go index c526249cc..d1bd3d559 100644 --- a/pkg/gateway/cache/gatewayclasses_trigger.go +++ b/pkg/gateway/cache/gatewayclasses_trigger.go @@ -1,60 +1,58 @@ package cache -import ( - gwv1 "sigs.k8s.io/gateway-api/apis/v1" - - "github.com/flomesh-io/fsm/pkg/gateway/utils" -) - // GatewayClassesTrigger is responsible for processing GatewayClass objects type GatewayClassesTrigger struct{} // Insert adds the GatewayClass object to the cache and returns true if the cache was modified func (p *GatewayClassesTrigger) Insert(obj interface{}, cache *GatewayCache) bool { - class, ok := obj.(*gwv1.GatewayClass) - if !ok { - log.Error().Msgf("unexpected object type %T", obj) - return false - } - - key := class.GetName() - - class, err := cache.informers.GetListers().GatewayClass.Get(key) - if err != nil { - log.Error().Msgf("Failed to get GatewayClass %s: %s", key, err) - return false - } - - if utils.IsEffectiveGatewayClass(class) { - cache.mutex.Lock() - defer cache.mutex.Unlock() - - cache.gatewayclass = class - return true - } - - return false + //class, ok := obj.(*gwv1.GatewayClass) + //if !ok { + // log.Error().Msgf("unexpected object type %T", obj) + // return false + //} + + //key := class.GetName() + + //class, err := cache.informers.GetListers().GatewayClass.Get(key) + //if err != nil { + // log.Error().Msgf("Failed to get GatewayClass %s: %s", key, err) + // return false + //} + + //if utils.IsEffectiveGatewayClass(class) { + // cache.mutex.Lock() + // defer cache.mutex.Unlock() + // + // cache.gatewayclass = class + // return true + //} + // + //return false + + return true } // Delete removes the GatewayClass object from the cache and returns true if the cache was modified func (p *GatewayClassesTrigger) Delete(obj interface{}, cache *GatewayCache) bool { - class, ok := obj.(*gwv1.GatewayClass) - if !ok { - log.Error().Msgf("unexpected object type %T", obj) - return false - } - - cache.mutex.Lock() - defer cache.mutex.Unlock() - - if cache.gatewayclass == nil { - return false - } - - if class.Name == cache.gatewayclass.Name { - cache.gatewayclass = nil - return true - } - - return false + //class, ok := obj.(*gwv1.GatewayClass) + //if !ok { + // log.Error().Msgf("unexpected object type %T", obj) + // return false + //} + + //cache.mutex.Lock() + //defer cache.mutex.Unlock() + // + //if cache.gatewayclass == nil { + // return false + //} + // + //if class.Name == cache.gatewayclass.Name { + // cache.gatewayclass = nil + // return true + //} + // + //return false + + return true } diff --git a/pkg/gateway/cache/gateways_trigger.go b/pkg/gateway/cache/gateways_trigger.go index a92b10b9a..938a78d47 100644 --- a/pkg/gateway/cache/gateways_trigger.go +++ b/pkg/gateway/cache/gateways_trigger.go @@ -17,13 +17,13 @@ func (p *GatewaysTrigger) Insert(obj interface{}, cache *GatewayCache) bool { return false } - key := utils.ObjectKey(gw) - - gw, err := cache.informers.GetListers().Gateway.Gateways(gw.Namespace).Get(gw.Name) - if err != nil { - log.Error().Msgf("Failed to get Gateway %s: %s", key, err) - return false - } + //key := utils.ObjectKey(gw) + // + //gw, err := cache.informers.GetListers().Gateway.Gateways(gw.NamespaceDerefOr).Get(gw.Name) + //if err != nil { + // log.Error().Msgf("Failed to get Gateway %s: %s", key, err) + // return false + //} return utils.IsActiveGateway(gw) } diff --git a/pkg/gateway/cache/grpcroute.go b/pkg/gateway/cache/grpcroute.go index 1a8afcdee..df79ecced 100644 --- a/pkg/gateway/cache/grpcroute.go +++ b/pkg/gateway/cache/grpcroute.go @@ -1,22 +1,54 @@ package cache import ( + "context" + + "github.com/flomesh-io/fsm/pkg/gateway/status/route" + + "k8s.io/apimachinery/pkg/fields" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/flomesh-io/fsm/pkg/constants" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" "github.com/flomesh-io/fsm/pkg/gateway/fgw" gwutils "github.com/flomesh-io/fsm/pkg/gateway/utils" ) -func (c *GatewayProcessor) processGRPCRoute(grpcRoute *gwv1.GRPCRoute) { - routePolicies := filterPoliciesByRoute(c.referenceGrants, c.policies, grpcRoute) - hostnameEnrichers := getHostnamePolicyEnrichers(routePolicies) +func (c *GatewayProcessor) processGRPCRoutes() { + list := &gwv1.GRPCRouteList{} + err := c.cache.client.List(context.Background(), list, &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(constants.GatewayGRPCRouteIndex, client.ObjectKeyFromObject(c.gateway).String()), + }) + if err != nil { + log.Error().Msgf("Failed to list GRPCRoutes: %v", err) + return + } - for _, ref := range grpcRoute.Spec.ParentRefs { - if !gwutils.IsRefToGateway(ref, gwutils.ObjectKey(c.gateway)) { + for _, grpcRoute := range gwutils.SortResources(gwutils.ToSlicePtr(list.Items)) { + c.processGRPCRoute(grpcRoute) + } +} + +func (c *GatewayProcessor) processGRPCRoute(grpcRoute *gwv1.GRPCRoute) { + hostnameEnrichers := c.getHostnamePolicyEnrichers(grpcRoute) + rsh := route.NewRouteStatusHolder( + grpcRoute, + &grpcRoute.ObjectMeta, + &grpcRoute.TypeMeta, + grpcRoute.Spec.Hostnames, + gwutils.ToSlicePtr(grpcRoute.Status.Parents), + ) + + for _, parentRef := range grpcRoute.Spec.ParentRefs { + if !gwutils.IsRefToGateway(parentRef, client.ObjectKeyFromObject(c.gateway)) { continue } - allowedListeners, _ := gwutils.GetAllowedListeners(c.getNamespaceLister(), c.gateway, ref, gwutils.ToRouteContext(grpcRoute), c.validListeners) + h := rsh.StatusUpdateFor(parentRef) + + allowedListeners := gwutils.GetAllowedListeners(c.cache.client, c.gateway, h) if len(allowedListeners) == 0 { continue } @@ -31,7 +63,7 @@ func (c *GatewayProcessor) processGRPCRoute(grpcRoute *gwv1.GRPCRoute) { grpcRule := fgw.L7RouteRule{} for _, hostname := range hostnames { - r := c.generateGRPCRouteCfg(grpcRoute, routePolicies) + r := c.generateGRPCRouteCfg(grpcRoute) for _, enricher := range hostnameEnrichers { enricher.Enrich(hostname, r) @@ -52,12 +84,12 @@ func (c *GatewayProcessor) processGRPCRoute(grpcRoute *gwv1.GRPCRoute) { } } -func (c *GatewayProcessor) generateGRPCRouteCfg(grpcRoute *gwv1.GRPCRoute, routePolicies routePolicies) *fgw.GRPCRouteRuleSpec { +func (c *GatewayProcessor) generateGRPCRouteCfg(grpcRoute *gwv1.GRPCRoute) *fgw.GRPCRouteRuleSpec { grpcSpec := &fgw.GRPCRouteRuleSpec{ RouteType: fgw.L7RouteTypeGRPC, Matches: make([]fgw.GRPCTrafficMatch, 0), } - enrichers := getGRPCRoutePolicyEnrichers(routePolicies) + enrichers := c.getGRPCRoutePolicyEnrichers(grpcRoute) for _, rule := range grpcRoute.Spec.Rules { backends := map[string]fgw.BackendServiceConfig{} diff --git a/pkg/gateway/cache/httproute.go b/pkg/gateway/cache/httproute.go index 4096ec88d..cdb3b02a8 100644 --- a/pkg/gateway/cache/httproute.go +++ b/pkg/gateway/cache/httproute.go @@ -1,22 +1,55 @@ package cache import ( + "context" + + "k8s.io/apimachinery/pkg/fields" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/flomesh-io/fsm/pkg/gateway/status/route" + + "github.com/flomesh-io/fsm/pkg/constants" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" "github.com/flomesh-io/fsm/pkg/gateway/fgw" gwutils "github.com/flomesh-io/fsm/pkg/gateway/utils" ) -func (c *GatewayProcessor) processHTTPRoute(httpRoute *gwv1.HTTPRoute) { - routePolicies := filterPoliciesByRoute(c.referenceGrants, c.policies, httpRoute) - hostnameEnrichers := getHostnamePolicyEnrichers(routePolicies) +func (c *GatewayProcessor) processHTTPRoutes() { + list := &gwv1.HTTPRouteList{} + err := c.cache.client.List(context.Background(), list, &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(constants.GatewayHTTPRouteIndex, client.ObjectKeyFromObject(c.gateway).String()), + }) + if err != nil { + log.Error().Msgf("Failed to list HTTPRoutes: %v", err) + return + } + + for _, httpRoute := range gwutils.SortResources(gwutils.ToSlicePtr(list.Items)) { + c.processHTTPRoute(httpRoute) + } +} - for _, ref := range httpRoute.Spec.ParentRefs { - if !gwutils.IsRefToGateway(ref, gwutils.ObjectKey(c.gateway)) { +func (c *GatewayProcessor) processHTTPRoute(httpRoute *gwv1.HTTPRoute) { + hostnameEnrichers := c.getHostnamePolicyEnrichers(httpRoute) + + rsh := route.NewRouteStatusUpdate( + httpRoute, + &httpRoute.ObjectMeta, + &httpRoute.TypeMeta, + httpRoute.Spec.Hostnames, + gwutils.ToSlicePtr(httpRoute.Status.Parents), + ) + + for _, parentRef := range httpRoute.Spec.ParentRefs { + if !gwutils.IsRefToGateway(parentRef, client.ObjectKeyFromObject(c.gateway)) { continue } - allowedListeners, _ := gwutils.GetAllowedListeners(c.getNamespaceLister(), c.gateway, ref, gwutils.ToRouteContext(httpRoute), c.validListeners) + h := rsh.StatusUpdateFor(parentRef) + + allowedListeners := gwutils.GetAllowedListeners(c.cache.client, c.gateway, h) log.Debug().Msgf("allowedListeners: %v", allowedListeners) if len(allowedListeners) == 0 { continue @@ -33,7 +66,7 @@ func (c *GatewayProcessor) processHTTPRoute(httpRoute *gwv1.HTTPRoute) { httpRule := fgw.L7RouteRule{} for _, hostname := range hostnames { - r := c.generateHTTPRouteConfig(httpRoute, routePolicies) + r := c.generateHTTPRouteConfig(httpRoute) for _, enricher := range hostnameEnrichers { enricher.Enrich(hostname, r) @@ -54,12 +87,12 @@ func (c *GatewayProcessor) processHTTPRoute(httpRoute *gwv1.HTTPRoute) { } } -func (c *GatewayProcessor) generateHTTPRouteConfig(httpRoute *gwv1.HTTPRoute, routePolicies routePolicies) *fgw.HTTPRouteRuleSpec { +func (c *GatewayProcessor) generateHTTPRouteConfig(httpRoute *gwv1.HTTPRoute) *fgw.HTTPRouteRuleSpec { httpSpec := &fgw.HTTPRouteRuleSpec{ RouteType: fgw.L7RouteTypeHTTP, Matches: make([]fgw.HTTPTrafficMatch, 0), } - enrichers := getHTTPRoutePolicyEnrichers(routePolicies) + enrichers := c.getHTTPRoutePolicyEnrichers(httpRoute) for _, rule := range httpRoute.Spec.Rules { backends := map[string]fgw.BackendServiceConfig{} @@ -113,6 +146,13 @@ func (c *GatewayProcessor) generateHTTPRouteConfig(httpRoute *gwv1.HTTPRoute, ro match.RequestParams = httpMatchQueryParams(m) } + if rule.Timeouts != nil { + match.Timeouts = &fgw.HTTPRouteTimeouts{ + Request: rule.Timeouts.Request, + BackendRequest: rule.Timeouts.BackendRequest, + } + } + for _, enricher := range enrichers { enricher.Enrich(m, match) } diff --git a/pkg/gateway/cache/methods.go b/pkg/gateway/cache/methods.go index 14dd52213..1a6a3797d 100644 --- a/pkg/gateway/cache/methods.go +++ b/pkg/gateway/cache/methods.go @@ -1,372 +1,38 @@ package cache import ( - "k8s.io/utils/ptr" + "context" "github.com/flomesh-io/fsm/pkg/k8s" - gwtypes "github.com/flomesh-io/fsm/pkg/gateway/types" - "github.com/flomesh-io/fsm/pkg/k8s/informers" - corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" - gwv1 "sigs.k8s.io/gateway-api/apis/v1" - gwv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" - - gwpav1alpha1 "github.com/flomesh-io/fsm/pkg/apis/policyattachment/v1alpha1" - "github.com/flomesh-io/fsm/pkg/constants" - - gwutils "github.com/flomesh-io/fsm/pkg/gateway/utils" ) -func (c *GatewayCache) getResourcesFromCache(resourceType informers.ResourceType, shouldSort bool) []client.Object { - return c.informers.GetGatewayResourcesFromCache(resourceType, shouldSort) -} - -func (c *GatewayCache) getActiveGateways() []*gwv1.Gateway { - //gateways := make([]*gwv1.Gateway, 0) - // - //for _, gw := range c.getResourcesFromCache(informers.GatewaysResourceType, false) { - // gw := gw.(*gwv1.Gateway) - // if gwutils.IsActiveGateway(gw) { - // gateways = append(gateways, gw) - // } - //} - // - //return gateways - - return gwutils.GetActiveGateways(c.getResourcesFromCache(informers.GatewaysResourceType, false)) -} - -// no need to check ReferenceGrant here -func (c *GatewayCache) isRoutableService(service client.ObjectKey) bool { - for _, checkRoutableFunc := range []func(client.ObjectKey) bool{ - c.isRoutableHTTPService, - c.isRoutableGRPCService, - c.isRoutableTLSService, - c.isRoutableTCPService, - c.isRoutableUDPService, - } { - if checkRoutableFunc(service) { - return true - } - } - - return false -} - -func (c *GatewayCache) isRoutableHTTPService(service client.ObjectKey) bool { - for _, r := range c.getResourcesFromCache(informers.HTTPRoutesResourceType, false) { - r := r.(*gwv1.HTTPRoute) - for _, rule := range r.Spec.Rules { - for _, backend := range rule.BackendRefs { - if c.isRefToService(r, backend.BackendObjectReference, service) { - return true - } - - for _, filter := range backend.Filters { - if filter.Type == gwv1.HTTPRouteFilterRequestMirror { - if c.isRefToService(r, filter.RequestMirror.BackendRef, service) { - return true - } - } - } - } - - for _, filter := range rule.Filters { - if filter.Type == gwv1.HTTPRouteFilterRequestMirror { - if c.isRefToService(r, filter.RequestMirror.BackendRef, service) { - return true - } - } - } - } - } - - return false -} - -func (c *GatewayCache) isRoutableGRPCService(service client.ObjectKey) bool { - for _, r := range c.getResourcesFromCache(informers.GRPCRoutesResourceType, false) { - r := r.(*gwv1.GRPCRoute) - for _, rule := range r.Spec.Rules { - for _, backend := range rule.BackendRefs { - if c.isRefToService(r, backend.BackendObjectReference, service) { - return true - } - - for _, filter := range backend.Filters { - if filter.Type == gwv1.GRPCRouteFilterRequestMirror { - if c.isRefToService(r, filter.RequestMirror.BackendRef, service) { - return true - } - } - } - } - - for _, filter := range rule.Filters { - if filter.Type == gwv1.GRPCRouteFilterRequestMirror { - if c.isRefToService(r, filter.RequestMirror.BackendRef, service) { - return true - } - } - } - } - } - - return false -} - -func (c *GatewayCache) isRoutableTLSService(service client.ObjectKey) bool { - for _, r := range c.getResourcesFromCache(informers.TLSRoutesResourceType, false) { - r := r.(*gwv1alpha2.TLSRoute) - for _, rule := range r.Spec.Rules { - for _, backend := range rule.BackendRefs { - if c.isRefToService(r, backend.BackendObjectReference, service) { - return true - } - } - } - } - - return false -} - -func (c *GatewayCache) isRoutableTCPService(service client.ObjectKey) bool { - for _, r := range c.getResourcesFromCache(informers.TCPRoutesResourceType, false) { - r := r.(*gwv1alpha2.TCPRoute) - for _, rule := range r.Spec.Rules { - for _, backend := range rule.BackendRefs { - if c.isRefToService(r, backend.BackendObjectReference, service) { - return true - } - } - } - } - - return false -} - -func (c *GatewayCache) isRoutableUDPService(service client.ObjectKey) bool { - for _, r := range c.getResourcesFromCache(informers.UDPRoutesResourceType, false) { - r := r.(*gwv1alpha2.UDPRoute) - for _, rule := range r.Spec.Rules { - for _, backend := range rule.BackendRefs { - if c.isRefToService(r, backend.BackendObjectReference, service) { - return true - } - } - } - } - - return false -} - -func (c *GatewayCache) isEffectiveRoute(parentRefs []gwv1.ParentReference) bool { - gateways := c.getActiveGateways() - - if len(gateways) == 0 { - return false - } - - for _, parentRef := range parentRefs { - for _, gw := range gateways { - if gwutils.IsRefToGateway(parentRef, client.ObjectKeyFromObject(gw)) { - return true - } - } - } - - return false -} - -// no need to check ReferenceGrant here -func (c *GatewayCache) isEffectiveTargetRef(policy client.Object, targetRef gwv1alpha2.NamespacedPolicyTargetReference) bool { - if targetRef.Group != constants.GatewayAPIGroup { - return false - } - - referenceGrants := c.getResourcesFromCache(informers.ReferenceGrantResourceType, false) - - switch targetRef.Kind { - case constants.GatewayAPIGatewayKind: - gateways := c.getActiveGateways() - if len(gateways) == 0 { - return false - } - - for _, gateway := range gateways { - if gwutils.IsRefToTarget(referenceGrants, policy, targetRef, gateway) { - return true - } - } - case constants.GatewayAPIHTTPRouteKind: - httproutes := c.getResourcesFromCache(informers.HTTPRoutesResourceType, false) - if len(httproutes) == 0 { - return false - } - - for _, route := range httproutes { - if gwutils.IsRefToTarget(referenceGrants, policy, targetRef, route) { - return true - } - } - case constants.GatewayAPIGRPCRouteKind: - grpcroutes := c.getResourcesFromCache(informers.GRPCRoutesResourceType, false) - if len(grpcroutes) == 0 { - return false - } - - for _, route := range grpcroutes { - if gwutils.IsRefToTarget(referenceGrants, policy, targetRef, route) { - return true - } - } - } - - return false -} - -// no need to check ReferenceGrant here -func (c *GatewayCache) isRoutableTargetService(owner client.Object, targetRef gwv1alpha2.NamespacedPolicyTargetReference) bool { - if (targetRef.Group == constants.KubernetesCoreGroup && targetRef.Kind == constants.KubernetesServiceKind) || - (targetRef.Group == constants.FlomeshMCSAPIGroup && targetRef.Kind == constants.FlomeshAPIServiceImportKind) { - return c.isRoutableService(client.ObjectKey{ - Namespace: gwutils.Namespace(targetRef.Namespace, owner.GetNamespace()), - Name: string(targetRef.Name), - }) - } - - return false -} - -// no need to check ReferenceGrant here -func (c *GatewayCache) isSecretReferred(secret client.ObjectKey) bool { - //ctx := context.TODO() - for _, gw := range c.getActiveGateways() { - for _, l := range gw.Spec.Listeners { - switch l.Protocol { - case gwv1.HTTPSProtocolType, gwv1.TLSProtocolType: - if l.TLS == nil { - continue - } - - if l.TLS.Mode == nil || *l.TLS.Mode == gwv1.TLSModeTerminate { - if len(l.TLS.CertificateRefs) > 0 { - for _, ref := range l.TLS.CertificateRefs { - if c.isRefToSecret(gw, ref, secret) { - return true - } - } - } - - if l.TLS.FrontendValidation != nil && len(l.TLS.FrontendValidation.CACertificateRefs) > 0 { - for _, ref := range l.TLS.FrontendValidation.CACertificateRefs { - ref := gwv1.SecretObjectReference{ - Group: ptr.To(ref.Group), - Kind: ptr.To(ref.Kind), - Name: ref.Name, - Namespace: ref.Namespace, - } - - if c.isRefToSecret(gw, ref, secret) { - return true - } - } - } - } - } - } - } - - for _, ut := range c.getResourcesFromCache(informers.UpstreamTLSPoliciesResourceType, false) { - ut := ut.(*gwpav1alpha1.UpstreamTLSPolicy) - - if ut.Spec.DefaultConfig != nil { - if c.isRefToSecret(ut, ut.Spec.DefaultConfig.CertificateRef, secret) { - return true - } - } - - if len(ut.Spec.Ports) > 0 { - for _, port := range ut.Spec.Ports { - if port.Config == nil { - continue - } - - if c.isRefToSecret(ut, port.Config.CertificateRef, secret) { - return true - } - } - } - } - - return false -} - -// no need to check ReferenceGrant here -func (c *GatewayCache) isConfigMapReferred(cm client.ObjectKey) bool { - //ctx := context.TODO() - for _, gw := range c.getActiveGateways() { - for _, l := range gw.Spec.Listeners { - switch l.Protocol { - case gwv1.HTTPSProtocolType, gwv1.TLSProtocolType: - if l.TLS == nil { - continue - } - - if l.TLS.Mode == nil || *l.TLS.Mode == gwv1.TLSModeTerminate { - if l.TLS.FrontendValidation == nil { - continue - } - - if len(l.TLS.FrontendValidation.CACertificateRefs) == 0 { - continue - } - - for _, ref := range l.TLS.FrontendValidation.CACertificateRefs { - if c.isRefToConfigMap(gw, ref, cm) { - return true - } - } - } - } - } - } - - return false -} - func (c *GatewayCache) getSecretFromCache(key client.ObjectKey) (*corev1.Secret, error) { - obj, err := c.informers.GetListers().Secret.Secrets(key.Namespace).Get(key.Name) - if err != nil { + obj := &corev1.Secret{} + if err := c.client.Get(context.TODO(), key, obj); err != nil { return nil, err } - obj.GetObjectKind().SetGroupVersionKind(constants.SecretGVK) - return obj, nil } func (c *GatewayCache) getConfigMapFromCache(key client.ObjectKey) (*corev1.ConfigMap, error) { - obj, err := c.informers.GetListers().ConfigMap.ConfigMaps(key.Namespace).Get(key.Name) - if err != nil { + obj := &corev1.ConfigMap{} + if err := c.client.Get(context.TODO(), key, obj); err != nil { return nil, err } - obj.GetObjectKind().SetGroupVersionKind(constants.ConfigMapGVK) - return obj, nil } func (c *GatewayCache) getServiceFromCache(key client.ObjectKey) (*corev1.Service, error) { - obj, err := c.informers.GetListers().Service.Services(key.Namespace).Get(key.Name) - if err != nil { + obj := &corev1.Service{} + if err := c.client.Get(context.TODO(), key, obj); err != nil { return nil, err } - obj.GetObjectKind().SetGroupVersionKind(constants.ServiceGVK) - return obj, nil } @@ -379,114 +45,3 @@ func (c *GatewayCache) isHeadlessService(key client.ObjectKey) bool { return k8s.IsHeadlessService(*service) } - -func (c *GatewayCache) isRefToService(referer client.Object, ref gwv1.BackendObjectReference, service client.ObjectKey) bool { - if !isValidBackendRefToGroupKindOfService(ref) { - log.Debug().Msgf("Unsupported backend group %s and kind %s for service", *ref.Group, *ref.Kind) - return false - } - - // fast-fail, not refer to the service with the same name - if string(ref.Name) != service.Name { - log.Debug().Msgf("Not refer to the service with the same name, ref.Name: %s, service.Name: %s", ref.Name, service.Name) - return false - } - - if ns := gwutils.Namespace(ref.Namespace, referer.GetNamespace()); ns != service.Namespace { - log.Debug().Msgf("Not refer to the service with the same namespace, resolved namespace: %s, service.Namespace: %s", ns, service.Namespace) - return false - } - - if ref.Namespace != nil && string(*ref.Namespace) == service.Namespace && string(*ref.Namespace) != referer.GetNamespace() { - gvk := referer.GetObjectKind().GroupVersionKind() - return gwutils.ValidCrossNamespaceRef( - c.getResourcesFromCache(informers.ReferenceGrantResourceType, false), - gwtypes.CrossNamespaceFrom{ - Group: gvk.Group, - Kind: gvk.Kind, - Namespace: referer.GetNamespace(), - }, - gwtypes.CrossNamespaceTo{ - Group: string(*ref.Group), - Kind: string(*ref.Kind), - Namespace: service.Namespace, - Name: service.Name, - }, - ) - } - - log.Debug().Msgf("Found a match, ref: %s/%s, service: %s/%s", gwutils.Namespace(ref.Namespace, referer.GetNamespace()), ref.Name, service.Namespace, service.Name) - return true -} - -func (c *GatewayCache) isRefToSecret(referer client.Object, ref gwv1.SecretObjectReference, secret client.ObjectKey) bool { - if !isValidRefToGroupKindOfSecret(ref) { - return false - } - - // fast-fail, not refer to the secret with the same name - if string(ref.Name) != secret.Name { - log.Debug().Msgf("Not refer to the secret with the same name, ref.Name: %s, secret.Name: %s", ref.Name, secret.Name) - return false - } - - if ns := gwutils.Namespace(ref.Namespace, referer.GetNamespace()); ns != secret.Namespace { - log.Debug().Msgf("Not refer to the secret with the same namespace, resolved namespace: %s, secret.Namespace: %s", ns, secret.Namespace) - return false - } - - if ref.Namespace != nil && string(*ref.Namespace) == secret.Namespace && string(*ref.Namespace) != referer.GetNamespace() { - return gwutils.ValidCrossNamespaceRef( - c.getResourcesFromCache(informers.ReferenceGrantResourceType, false), - gwtypes.CrossNamespaceFrom{ - Group: referer.GetObjectKind().GroupVersionKind().Group, - Kind: referer.GetObjectKind().GroupVersionKind().Kind, - Namespace: referer.GetNamespace(), - }, - gwtypes.CrossNamespaceTo{ - Group: corev1.GroupName, - Kind: constants.KubernetesSecretKind, - Namespace: secret.Namespace, - Name: secret.Name, - }, - ) - } - - return true -} - -func (c *GatewayCache) isRefToConfigMap(referer client.Object, ref gwv1.ObjectReference, cm client.ObjectKey) bool { - if !isValidRefToGroupKindOfConfigMap(ref) { - return false - } - - // fast-fail, not refer to the cm with the same name - if string(ref.Name) != cm.Name { - log.Debug().Msgf("Not refer to the cm with the same name, ref.Name: %s, cm.Name: %s", ref.Name, cm.Name) - return false - } - - if ns := gwutils.Namespace(ref.Namespace, referer.GetNamespace()); ns != cm.Namespace { - log.Debug().Msgf("Not refer to the cm with the same namespace, resolved namespace: %s, cm.Namespace: %s", ns, cm.Namespace) - return false - } - - if ref.Namespace != nil && string(*ref.Namespace) == cm.Namespace && string(*ref.Namespace) != referer.GetNamespace() { - return gwutils.ValidCrossNamespaceRef( - c.getResourcesFromCache(informers.ReferenceGrantResourceType, false), - gwtypes.CrossNamespaceFrom{ - Group: referer.GetObjectKind().GroupVersionKind().Group, - Kind: referer.GetObjectKind().GroupVersionKind().Kind, - Namespace: referer.GetNamespace(), - }, - gwtypes.CrossNamespaceTo{ - Group: corev1.GroupName, - Kind: constants.KubernetesConfigMapKind, - Namespace: cm.Namespace, - Name: cm.Name, - }, - ) - } - - return true -} diff --git a/pkg/gateway/cache/policies.go b/pkg/gateway/cache/policies.go index 66e477f20..bb8df65b8 100644 --- a/pkg/gateway/cache/policies.go +++ b/pkg/gateway/cache/policies.go @@ -1,455 +1,79 @@ package cache import ( - gwpkg "github.com/flomesh-io/fsm/pkg/gateway/types" - "github.com/flomesh-io/fsm/pkg/k8s/informers" + "fmt" - "github.com/flomesh-io/fsm/pkg/gateway/policy/utils/retry" + corev1 "k8s.io/api/core/v1" - "github.com/flomesh-io/fsm/pkg/gateway/policy/utils/sessionsticky" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" - "github.com/flomesh-io/fsm/pkg/gateway/policy/utils/loadbalancer" - - "github.com/flomesh-io/fsm/pkg/gateway/policy/utils/circuitbreaking" - - "github.com/flomesh-io/fsm/pkg/gateway/policy/utils/healthcheck" - "github.com/flomesh-io/fsm/pkg/gateway/policy/utils/upstreamtls" + "k8s.io/apimachinery/pkg/fields" "github.com/flomesh-io/fsm/pkg/constants" "github.com/flomesh-io/fsm/pkg/gateway/policy" "sigs.k8s.io/controller-runtime/pkg/client" - - gwpav1alpha1 "github.com/flomesh-io/fsm/pkg/apis/policyattachment/v1alpha1" - gwutils "github.com/flomesh-io/fsm/pkg/gateway/utils" ) -func (c *GatewayCache) policyAttachments() globalPolicyAttachments { - return globalPolicyAttachments{ - rateLimits: c.rateLimits(), - accessControls: c.accessControls(), - faultInjections: c.faultInjections(), - } -} +func (c *GatewayProcessor) getPortPolicyEnrichers(gateway *gwv1.Gateway) []policy.PortPolicyEnricher { + cc := c.cache.client + key := client.ObjectKeyFromObject(gateway).String() + selector := fields.OneTermEqualSelector(constants.PortPolicyAttachmentIndex, key) -func (c *GatewayProcessor) getPortPolicyEnrichers() []policy.PortPolicyEnricher { return []policy.PortPolicyEnricher{ - &policy.RateLimitPortEnricher{Data: c.policies.rateLimits[gwpkg.PolicyMatchTypePort], ReferenceGrants: c.referenceGrants}, - &policy.AccessControlPortEnricher{Data: c.policies.accessControls[gwpkg.PolicyMatchTypePort], ReferenceGrants: c.referenceGrants}, - } -} - -func getHostnamePolicyEnrichers(routePolicies routePolicies) []policy.HostnamePolicyEnricher { - return []policy.HostnamePolicyEnricher{ - &policy.RateLimitHostnameEnricher{Data: routePolicies.hostnamesRateLimits}, - &policy.AccessControlHostnameEnricher{Data: routePolicies.hostnamesAccessControls}, - &policy.FaultInjectionHostnameEnricher{Data: routePolicies.hostnamesFaultInjections}, - } -} - -func getHTTPRoutePolicyEnrichers(routePolicies routePolicies) []policy.HTTPRoutePolicyEnricher { - return []policy.HTTPRoutePolicyEnricher{ - &policy.RateLimitHTTPRouteEnricher{Data: routePolicies.httpRouteRateLimits}, - &policy.AccessControlHTTPRouteEnricher{Data: routePolicies.httpRouteAccessControls}, - &policy.FaultInjectionHTTPRouteEnricher{Data: routePolicies.httpRouteFaultInjections}, - } -} - -func getGRPCRoutePolicyEnrichers(routePolicies routePolicies) []policy.GRPCRoutePolicyEnricher { - return []policy.GRPCRoutePolicyEnricher{ - &policy.RateLimitGRPCRouteEnricher{Data: routePolicies.grpcRouteRateLimits}, - &policy.AccessControlGRPCRouteEnricher{Data: routePolicies.grpcRouteAccessControls}, - &policy.FaultInjectionGRPCRouteEnricher{Data: routePolicies.grpcRouteFaultInjections}, - } -} - -func (c *GatewayProcessor) getServicePolicyEnrichers() []policy.ServicePolicyEnricher { - return []policy.ServicePolicyEnricher{ - &policy.SessionStickyPolicyEnricher{Data: c.sessionStickies()}, - &policy.LoadBalancerPolicyEnricher{Data: c.loadBalancers()}, - &policy.CircuitBreakingPolicyEnricher{Data: c.circuitBreakings()}, - &policy.HealthCheckPolicyEnricher{Data: c.healthChecks()}, - &policy.UpstreamTLSPolicyEnricher{Data: c.upstreamTLS()}, - &policy.RetryPolicyEnricher{Data: c.retryConfigs()}, - } -} - -func (c *GatewayCache) rateLimits() map[gwpkg.PolicyMatchType][]gwpav1alpha1.RateLimitPolicy { - rateLimits := make(map[gwpkg.PolicyMatchType][]gwpav1alpha1.RateLimitPolicy) - for _, matchType := range []gwpkg.PolicyMatchType{ - gwpkg.PolicyMatchTypePort, - gwpkg.PolicyMatchTypeHostnames, - gwpkg.PolicyMatchTypeHTTPRoute, - gwpkg.PolicyMatchTypeGRPCRoute, - } { - rateLimits[matchType] = make([]gwpav1alpha1.RateLimitPolicy, 0) - } - - for _, p := range c.getResourcesFromCache(informers.RateLimitPoliciesResourceType, true) { - p := p.(*gwpav1alpha1.RateLimitPolicy) - - if gwutils.IsAcceptedPolicyAttachment(p.Status.Conditions) { - spec := p.Spec - targetRef := spec.TargetRef - - switch { - case gwutils.IsTargetRefToGVK(targetRef, constants.GatewayGVK) && len(spec.Ports) > 0: - rateLimits[gwpkg.PolicyMatchTypePort] = append(rateLimits[gwpkg.PolicyMatchTypePort], *p) - case (gwutils.IsTargetRefToGVK(targetRef, constants.HTTPRouteGVK) || gwutils.IsTargetRefToGVK(targetRef, constants.GRPCRouteGVK)) && len(spec.Hostnames) > 0: - rateLimits[gwpkg.PolicyMatchTypeHostnames] = append(rateLimits[gwpkg.PolicyMatchTypeHostnames], *p) - case gwutils.IsTargetRefToGVK(targetRef, constants.HTTPRouteGVK) && len(spec.HTTPRateLimits) > 0: - rateLimits[gwpkg.PolicyMatchTypeHTTPRoute] = append(rateLimits[gwpkg.PolicyMatchTypeHTTPRoute], *p) - case gwutils.IsTargetRefToGVK(targetRef, constants.GRPCRouteGVK) && len(spec.GRPCRateLimits) > 0: - rateLimits[gwpkg.PolicyMatchTypeGRPCRoute] = append(rateLimits[gwpkg.PolicyMatchTypeGRPCRoute], *p) - } - } - } - - return rateLimits -} - -func (c *GatewayCache) accessControls() map[gwpkg.PolicyMatchType][]gwpav1alpha1.AccessControlPolicy { - accessControls := make(map[gwpkg.PolicyMatchType][]gwpav1alpha1.AccessControlPolicy) - for _, matchType := range []gwpkg.PolicyMatchType{ - gwpkg.PolicyMatchTypePort, - gwpkg.PolicyMatchTypeHostnames, - gwpkg.PolicyMatchTypeHTTPRoute, - gwpkg.PolicyMatchTypeGRPCRoute, - } { - accessControls[matchType] = make([]gwpav1alpha1.AccessControlPolicy, 0) - } - - for _, p := range c.getResourcesFromCache(informers.AccessControlPoliciesResourceType, true) { - p := p.(*gwpav1alpha1.AccessControlPolicy) - - if gwutils.IsAcceptedPolicyAttachment(p.Status.Conditions) { - spec := p.Spec - targetRef := spec.TargetRef - - switch { - case gwutils.IsTargetRefToGVK(targetRef, constants.GatewayGVK) && len(spec.Ports) > 0: - accessControls[gwpkg.PolicyMatchTypePort] = append(accessControls[gwpkg.PolicyMatchTypePort], *p) - case (gwutils.IsTargetRefToGVK(targetRef, constants.HTTPRouteGVK) || gwutils.IsTargetRefToGVK(targetRef, constants.GRPCRouteGVK)) && len(spec.Hostnames) > 0: - accessControls[gwpkg.PolicyMatchTypeHostnames] = append(accessControls[gwpkg.PolicyMatchTypeHostnames], *p) - case gwutils.IsTargetRefToGVK(targetRef, constants.HTTPRouteGVK) && len(spec.HTTPAccessControls) > 0: - accessControls[gwpkg.PolicyMatchTypeHTTPRoute] = append(accessControls[gwpkg.PolicyMatchTypeHTTPRoute], *p) - case gwutils.IsTargetRefToGVK(targetRef, constants.GRPCRouteGVK) && len(spec.GRPCAccessControls) > 0: - accessControls[gwpkg.PolicyMatchTypeGRPCRoute] = append(accessControls[gwpkg.PolicyMatchTypeGRPCRoute], *p) - } - } - } - - return accessControls -} - -func (c *GatewayCache) faultInjections() map[gwpkg.PolicyMatchType][]gwpav1alpha1.FaultInjectionPolicy { - faultInjections := make(map[gwpkg.PolicyMatchType][]gwpav1alpha1.FaultInjectionPolicy) - for _, matchType := range []gwpkg.PolicyMatchType{ - gwpkg.PolicyMatchTypeHostnames, - gwpkg.PolicyMatchTypeHTTPRoute, - gwpkg.PolicyMatchTypeGRPCRoute, - } { - faultInjections[matchType] = make([]gwpav1alpha1.FaultInjectionPolicy, 0) - } - - for _, p := range c.getResourcesFromCache(informers.FaultInjectionPoliciesResourceType, true) { - p := p.(*gwpav1alpha1.FaultInjectionPolicy) - - if gwutils.IsAcceptedPolicyAttachment(p.Status.Conditions) { - spec := p.Spec - targetRef := spec.TargetRef - - switch { - case (gwutils.IsTargetRefToGVK(targetRef, constants.HTTPRouteGVK) || gwutils.IsTargetRefToGVK(targetRef, constants.GRPCRouteGVK)) && len(spec.Hostnames) > 0: - faultInjections[gwpkg.PolicyMatchTypeHostnames] = append(faultInjections[gwpkg.PolicyMatchTypeHostnames], *p) - case gwutils.IsTargetRefToGVK(targetRef, constants.HTTPRouteGVK) && len(spec.HTTPFaultInjections) > 0: - faultInjections[gwpkg.PolicyMatchTypeHTTPRoute] = append(faultInjections[gwpkg.PolicyMatchTypeHTTPRoute], *p) - case gwutils.IsTargetRefToGVK(targetRef, constants.GRPCRouteGVK) && len(spec.GRPCFaultInjections) > 0: - faultInjections[gwpkg.PolicyMatchTypeGRPCRoute] = append(faultInjections[gwpkg.PolicyMatchTypeGRPCRoute], *p) - } - } - } - - return faultInjections -} - -func filterPoliciesByRoute(referenceGrants []client.Object, policies globalPolicyAttachments, route client.Object) routePolicies { - result := routePolicies{ - hostnamesRateLimits: make([]gwpav1alpha1.RateLimitPolicy, 0), - httpRouteRateLimits: make([]gwpav1alpha1.RateLimitPolicy, 0), - grpcRouteRateLimits: make([]gwpav1alpha1.RateLimitPolicy, 0), - hostnamesAccessControls: make([]gwpav1alpha1.AccessControlPolicy, 0), - httpRouteAccessControls: make([]gwpav1alpha1.AccessControlPolicy, 0), - grpcRouteAccessControls: make([]gwpav1alpha1.AccessControlPolicy, 0), - hostnamesFaultInjections: make([]gwpav1alpha1.FaultInjectionPolicy, 0), - httpRouteFaultInjections: make([]gwpav1alpha1.FaultInjectionPolicy, 0), - grpcRouteFaultInjections: make([]gwpav1alpha1.FaultInjectionPolicy, 0), - } - - if len(policies.rateLimits[gwpkg.PolicyMatchTypeHostnames]) > 0 { - for _, rateLimit := range policies.rateLimits[gwpkg.PolicyMatchTypeHostnames] { - rateLimit := rateLimit - if gwutils.IsRefToTarget(referenceGrants, &rateLimit, rateLimit.Spec.TargetRef, route) { - result.hostnamesRateLimits = append(result.hostnamesRateLimits, rateLimit) - } - } - } - - if len(policies.rateLimits[gwpkg.PolicyMatchTypeHTTPRoute]) > 0 { - for _, rateLimit := range policies.rateLimits[gwpkg.PolicyMatchTypeHTTPRoute] { - rateLimit := rateLimit - if gwutils.IsRefToTarget(referenceGrants, &rateLimit, rateLimit.Spec.TargetRef, route) { - result.httpRouteRateLimits = append(result.httpRouteRateLimits, rateLimit) - } - } - } - - if len(policies.rateLimits[gwpkg.PolicyMatchTypeGRPCRoute]) > 0 { - for _, rateLimit := range policies.rateLimits[gwpkg.PolicyMatchTypeGRPCRoute] { - rateLimit := rateLimit - if gwutils.IsRefToTarget(referenceGrants, &rateLimit, rateLimit.Spec.TargetRef, route) { - result.grpcRouteRateLimits = append(result.grpcRouteRateLimits, rateLimit) - } - } - } - - if len(policies.accessControls[gwpkg.PolicyMatchTypeHostnames]) > 0 { - for _, ac := range policies.accessControls[gwpkg.PolicyMatchTypeHostnames] { - ac := ac - if gwutils.IsRefToTarget(referenceGrants, &ac, ac.Spec.TargetRef, route) { - result.hostnamesAccessControls = append(result.hostnamesAccessControls, ac) - } - } - } - - if len(policies.accessControls[gwpkg.PolicyMatchTypeHTTPRoute]) > 0 { - for _, ac := range policies.accessControls[gwpkg.PolicyMatchTypeHTTPRoute] { - ac := ac - if gwutils.IsRefToTarget(referenceGrants, &ac, ac.Spec.TargetRef, route) { - result.httpRouteAccessControls = append(result.httpRouteAccessControls, ac) - } - } - } - - if len(policies.accessControls[gwpkg.PolicyMatchTypeGRPCRoute]) > 0 { - for _, ac := range policies.accessControls[gwpkg.PolicyMatchTypeGRPCRoute] { - ac := ac - if gwutils.IsRefToTarget(referenceGrants, &ac, ac.Spec.TargetRef, route) { - result.grpcRouteAccessControls = append(result.grpcRouteAccessControls, ac) - } - } - } - - if len(policies.faultInjections[gwpkg.PolicyMatchTypeHostnames]) > 0 { - for _, fj := range policies.faultInjections[gwpkg.PolicyMatchTypeHostnames] { - fj := fj - if gwutils.IsRefToTarget(referenceGrants, &fj, fj.Spec.TargetRef, route) { - result.hostnamesFaultInjections = append(result.hostnamesFaultInjections, fj) - } - } - } - - if len(policies.faultInjections[gwpkg.PolicyMatchTypeHTTPRoute]) > 0 { - for _, fj := range policies.faultInjections[gwpkg.PolicyMatchTypeHTTPRoute] { - fj := fj - if gwutils.IsRefToTarget(referenceGrants, &fj, fj.Spec.TargetRef, route) { - result.httpRouteFaultInjections = append(result.httpRouteFaultInjections, fj) - } - } - } - - if len(policies.faultInjections[gwpkg.PolicyMatchTypeGRPCRoute]) > 0 { - for _, fj := range policies.faultInjections[gwpkg.PolicyMatchTypeGRPCRoute] { - fj := fj - if gwutils.IsRefToTarget(referenceGrants, &fj, fj.Spec.TargetRef, route) { - result.grpcRouteFaultInjections = append(result.grpcRouteFaultInjections, fj) - } - } - } - - return result -} - -func (c *GatewayProcessor) sessionStickies() map[string]*gwpav1alpha1.SessionStickyConfig { - sessionStickies := make(map[string]*gwpav1alpha1.SessionStickyConfig) - - for _, sessionSticky := range c.getResourcesFromCache(informers.SessionStickyPoliciesResourceType, true) { - sessionSticky := sessionSticky.(*gwpav1alpha1.SessionStickyPolicy) - - if gwutils.IsAcceptedPolicyAttachment(sessionSticky.Status.Conditions) { - for _, p := range sessionSticky.Spec.Ports { - if svcPortName := c.targetRefToServicePortName(sessionSticky, sessionSticky.Spec.TargetRef, int32(p.Port)); svcPortName != nil { - cfg := sessionsticky.ComputeSessionStickyConfig(p.Config, sessionSticky.Spec.DefaultConfig) - - if cfg == nil { - continue - } - - if _, ok := sessionStickies[svcPortName.String()]; ok { - log.Warn().Msgf("Policy is already defined for service port %s, SessionStickyPolicy %s/%s:%d will be dropped", svcPortName.String(), sessionSticky.Namespace, sessionSticky.Name, p.Port) - continue - } - - sessionStickies[svcPortName.String()] = cfg - } - } - } + policy.NewRateLimitPortEnricher(cc, selector), + policy.NewAccessControlPortEnricher(cc, selector), } - - return sessionStickies } -func (c *GatewayProcessor) loadBalancers() map[string]*gwpav1alpha1.LoadBalancerType { - loadBalancers := make(map[string]*gwpav1alpha1.LoadBalancerType) +func (c *GatewayProcessor) getHostnamePolicyEnrichers(route client.Object) []policy.HostnamePolicyEnricher { + cc := c.cache.client + key := fmt.Sprintf("%s/%s/%s", route.GetObjectKind().GroupVersionKind().Kind, route.GetNamespace(), route.GetName()) + selector := fields.OneTermEqualSelector(constants.HostnamePolicyAttachmentIndex, key) - for _, lb := range c.getResourcesFromCache(informers.LoadBalancerPoliciesResourceType, true) { - lb := lb.(*gwpav1alpha1.LoadBalancerPolicy) - - if gwutils.IsAcceptedPolicyAttachment(lb.Status.Conditions) { - for _, p := range lb.Spec.Ports { - if svcPortName := c.targetRefToServicePortName(lb, lb.Spec.TargetRef, int32(p.Port)); svcPortName != nil { - t := loadbalancer.ComputeLoadBalancerType(p.Type, lb.Spec.DefaultType) - - if t == nil { - continue - } - - if _, ok := loadBalancers[svcPortName.String()]; ok { - log.Warn().Msgf("Policy is already defined for service port %s, LoadBalancerPolicy %s/%s:%d will be dropped", svcPortName.String(), lb.Namespace, lb.Name, p.Port) - continue - } - - loadBalancers[svcPortName.String()] = t - } - } - } - } - - return loadBalancers -} - -func (c *GatewayProcessor) circuitBreakings() map[string]*gwpav1alpha1.CircuitBreakingConfig { - configs := make(map[string]*gwpav1alpha1.CircuitBreakingConfig) - - for _, circuitBreaking := range c.getResourcesFromCache(informers.CircuitBreakingPoliciesResourceType, true) { - circuitBreaking := circuitBreaking.(*gwpav1alpha1.CircuitBreakingPolicy) - - if gwutils.IsAcceptedPolicyAttachment(circuitBreaking.Status.Conditions) { - for _, p := range circuitBreaking.Spec.Ports { - if svcPortName := c.targetRefToServicePortName(circuitBreaking, circuitBreaking.Spec.TargetRef, int32(p.Port)); svcPortName != nil { - cfg := circuitbreaking.ComputeCircuitBreakingConfig(p.Config, circuitBreaking.Spec.DefaultConfig) - - if cfg == nil { - continue - } - - if _, ok := configs[svcPortName.String()]; ok { - log.Warn().Msgf("Policy is already defined for service port %s, CircuitBreakingPolicy %s/%s:%d will be dropped", svcPortName.String(), circuitBreaking.Namespace, circuitBreaking.Name, p.Port) - continue - } - - configs[svcPortName.String()] = cfg - } - } - } + return []policy.HostnamePolicyEnricher{ + policy.NewRateLimitHostnameEnricher(cc, selector), + policy.NewAccessControlHostnameEnricher(cc, selector), + policy.NewFaultInjectionHostnameEnricher(cc, selector), } - - return configs } -func (c *GatewayProcessor) healthChecks() map[string]*gwpav1alpha1.HealthCheckConfig { - configs := make(map[string]*gwpav1alpha1.HealthCheckConfig) - - for _, healthCheck := range c.getResourcesFromCache(informers.HealthCheckPoliciesResourceType, true) { - healthCheck := healthCheck.(*gwpav1alpha1.HealthCheckPolicy) - - if gwutils.IsAcceptedPolicyAttachment(healthCheck.Status.Conditions) { - for _, p := range healthCheck.Spec.Ports { - if svcPortName := c.targetRefToServicePortName(healthCheck, healthCheck.Spec.TargetRef, int32(p.Port)); svcPortName != nil { - cfg := healthcheck.ComputeHealthCheckConfig(p.Config, healthCheck.Spec.DefaultConfig) - - if cfg == nil { - continue - } +func (c *GatewayProcessor) getHTTPRoutePolicyEnrichers(route *gwv1.HTTPRoute) []policy.HTTPRoutePolicyEnricher { + cc := c.cache.client + key := client.ObjectKeyFromObject(route).String() + selector := fields.OneTermEqualSelector(constants.HTTPRoutePolicyAttachmentIndex, key) - if _, ok := configs[svcPortName.String()]; ok { - log.Warn().Msgf("Policy is already defined for service port %s, HealthCheckPolicy %s/%s:%d will be dropped", svcPortName.String(), healthCheck.Namespace, healthCheck.Name, p.Port) - continue - } - - configs[svcPortName.String()] = cfg - } - } - } + return []policy.HTTPRoutePolicyEnricher{ + policy.NewRateLimitHTTPRouteEnricher(cc, selector), + policy.NewAccessControlHTTPRouteEnricher(cc, selector), + policy.NewFaultInjectionHTTPRouteEnricher(cc, selector), } - - return configs } -func (c *GatewayProcessor) upstreamTLS() map[string]*policy.UpstreamTLSConfig { - configs := make(map[string]*policy.UpstreamTLSConfig) +func (c *GatewayProcessor) getGRPCRoutePolicyEnrichers(route *gwv1.GRPCRoute) []policy.GRPCRoutePolicyEnricher { + cc := c.cache.client + key := client.ObjectKeyFromObject(route).String() + selector := fields.OneTermEqualSelector(constants.GRPCRoutePolicyAttachmentIndex, key) - for _, upstreamTLS := range c.getResourcesFromCache(informers.UpstreamTLSPoliciesResourceType, true) { - upstreamTLS := upstreamTLS.(*gwpav1alpha1.UpstreamTLSPolicy) - - if gwutils.IsAcceptedPolicyAttachment(upstreamTLS.Status.Conditions) { - for _, p := range upstreamTLS.Spec.Ports { - if svcPortName := c.targetRefToServicePortName(upstreamTLS, upstreamTLS.Spec.TargetRef, int32(p.Port)); svcPortName != nil { - cfg := upstreamtls.ComputeUpstreamTLSConfig(p.Config, upstreamTLS.Spec.DefaultConfig) - - if cfg == nil { - continue - } - - secret, err := c.secretRefToSecret(upstreamTLS, cfg.CertificateRef) - if err != nil { - log.Error().Msgf("Failed to resolve Secret: %s", err) - continue - } - - if _, ok := configs[svcPortName.String()]; ok { - log.Warn().Msgf("Policy is already defined for service port %s, UpstreamTLSPolicy %s/%s:%d will be dropped", svcPortName.String(), upstreamTLS.Namespace, upstreamTLS.Name, p.Port) - continue - } - - configs[svcPortName.String()] = &policy.UpstreamTLSConfig{ - MTLS: cfg.MTLS, - Secret: secret, - } - } - } - } + return []policy.GRPCRoutePolicyEnricher{ + policy.NewRateLimitGRPCRouteEnricher(cc, selector), + policy.NewAccessControlGRPCRouteEnricher(cc, selector), + policy.NewFaultInjectionGRPCRouteEnricher(cc, selector), } - - return configs } -func (c *GatewayProcessor) retryConfigs() map[string]*gwpav1alpha1.RetryConfig { - configs := make(map[string]*gwpav1alpha1.RetryConfig) +func (c *GatewayProcessor) getServicePolicyEnrichers(svc *corev1.Service) []policy.ServicePolicyEnricher { + cc := c.cache.client + key := client.ObjectKeyFromObject(svc).String() + selector := fields.OneTermEqualSelector(constants.ServicePolicyAttachmentIndex, key) - for _, retryPolicy := range c.getResourcesFromCache(informers.RetryPoliciesResourceType, true) { - retryPolicy := retryPolicy.(*gwpav1alpha1.RetryPolicy) - - if gwutils.IsAcceptedPolicyAttachment(retryPolicy.Status.Conditions) { - for _, p := range retryPolicy.Spec.Ports { - if svcPortName := c.targetRefToServicePortName(retryPolicy, retryPolicy.Spec.TargetRef, int32(p.Port)); svcPortName != nil { - cfg := retry.ComputeRetryConfig(p.Config, retryPolicy.Spec.DefaultConfig) - - if cfg == nil { - continue - } - - if _, ok := configs[svcPortName.String()]; ok { - log.Warn().Msgf("Policy is already defined for service port %s, RetryPolicy %s/%s:%d will be dropped", svcPortName.String(), retryPolicy.Namespace, retryPolicy.Name, p.Port) - continue - } - - configs[svcPortName.String()] = cfg - } - } - } + return []policy.ServicePolicyEnricher{ + policy.NewSessionStickyPolicyEnricher(cc, selector, c.targetRefToServicePortName), + policy.NewLoadBalancerPolicyEnricher(cc, selector, c.targetRefToServicePortName), + policy.NewCircuitBreakingPolicyEnricher(cc, selector, c.targetRefToServicePortName), + policy.NewUpstreamTLSPolicyEnricher(cc, selector, c.targetRefToServicePortName, c.secretRefToSecret), + policy.NewRetryPolicyEnricher(cc, selector, c.targetRefToServicePortName), + policy.NewHealthCheckPolicyEnricher(cc, selector, c.targetRefToServicePortName), } - - return configs } diff --git a/pkg/gateway/cache/processor.go b/pkg/gateway/cache/processor.go index 84be789fe..3263e7453 100644 --- a/pkg/gateway/cache/processor.go +++ b/pkg/gateway/cache/processor.go @@ -1,6 +1,7 @@ package cache import ( + "context" "fmt" "github.com/flomesh-io/fsm/pkg/k8s" @@ -14,8 +15,6 @@ import ( "github.com/flomesh-io/fsm/pkg/configurator" - v1 "k8s.io/client-go/listers/core/v1" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/pointer" @@ -26,30 +25,25 @@ import ( "github.com/flomesh-io/fsm/pkg/constants" "github.com/flomesh-io/fsm/pkg/gateway/fgw" gwtypes "github.com/flomesh-io/fsm/pkg/gateway/types" - "github.com/flomesh-io/fsm/pkg/k8s/informers" "github.com/flomesh-io/fsm/pkg/utils" ) type GatewayProcessor struct { - cache *GatewayCache - gateway *gwv1.Gateway - policies globalPolicyAttachments - referenceGrants []client.Object - validListeners []gwtypes.Listener - services map[string]serviceContext - rules map[int32]fgw.RouteRule - upstreams calculateEndpointsFunc + cache *GatewayCache + gateway *gwv1.Gateway + validListeners []gwtypes.Listener + services map[string]serviceContext + rules map[int32]fgw.RouteRule + upstreams calculateEndpointsFunc } -func NewGatewayProcessor(cache *GatewayCache, gateway *gwv1.Gateway, policies globalPolicyAttachments, referenceGrants []client.Object) *GatewayProcessor { +func NewGatewayProcessor(cache *GatewayCache, gateway *gwv1.Gateway) *GatewayProcessor { p := &GatewayProcessor{ - cache: cache, - gateway: gateway, - policies: policies, - referenceGrants: referenceGrants, - validListeners: gwutils.GetValidListenersForGateway(gateway), - services: make(map[string]serviceContext), - rules: make(map[int32]fgw.RouteRule), + cache: cache, + gateway: gateway, + validListeners: gwutils.GetValidListenersForGateway(gateway), + services: make(map[string]serviceContext), + rules: make(map[int32]fgw.RouteRule), } if cache.useEndpointSlices { @@ -79,14 +73,6 @@ func (c *GatewayProcessor) build() *fgw.ConfigSpec { return configSpec } -func (c *GatewayProcessor) getResourcesFromCache(resourceType informers.ResourceType, shouldSort bool) []client.Object { - return c.cache.getResourcesFromCache(resourceType, shouldSort) -} - -func (c *GatewayProcessor) getNamespaceLister() v1.NamespaceLister { - return c.cache.informers.GetListers().Namespace -} - func (c *GatewayProcessor) getConfig() configurator.Configurator { return c.cache.cfg } @@ -102,7 +88,7 @@ func (c *GatewayProcessor) isDebugEnabled() bool { func (c *GatewayProcessor) listeners() []fgw.Listener { listeners := make([]fgw.Listener, 0) - enrichers := c.getPortPolicyEnrichers() + enrichers := c.getPortPolicyEnrichers(c.gateway) for _, l := range c.validListeners { listener := &fgw.Listener{ @@ -233,37 +219,17 @@ func (c *GatewayProcessor) caCerts(l gwtypes.Listener) []string { } func (c *GatewayProcessor) routeRules() map[int32]fgw.RouteRule { - for _, httpRoute := range c.getResourcesFromCache(informers.HTTPRoutesResourceType, true) { - httpRoute := httpRoute.(*gwv1.HTTPRoute) - c.processHTTPRoute(httpRoute) - } - - for _, grpcRoute := range c.getResourcesFromCache(informers.GRPCRoutesResourceType, true) { - grpcRoute := grpcRoute.(*gwv1.GRPCRoute) - c.processGRPCRoute(grpcRoute) - } - - for _, tlsRoute := range c.getResourcesFromCache(informers.TLSRoutesResourceType, true) { - tlsRoute := tlsRoute.(*gwv1alpha2.TLSRoute) - c.processTLSRoute(tlsRoute) - } - - for _, tcpRoute := range c.getResourcesFromCache(informers.TCPRoutesResourceType, true) { - tcpRoute := tcpRoute.(*gwv1alpha2.TCPRoute) - c.processTCPRoute(tcpRoute) - } - - for _, udpRoute := range c.getResourcesFromCache(informers.UDPRoutesResourceType, true) { - udpRoute := udpRoute.(*gwv1alpha2.UDPRoute) - c.processUDPRoute(udpRoute) - } + c.processHTTPRoutes() + c.processGRPCRoutes() + c.processTLSRoutes() + c.processTCPRoutes() + c.processUDPRoutes() return c.rules } func (c *GatewayProcessor) serviceConfigs() map[string]fgw.ServiceConfig { configs := make(map[string]fgw.ServiceConfig) - enrichers := c.getServicePolicyEnrichers() for svcPortName, svcInfo := range c.services { svcKey := svcInfo.svcPortName.NamespacedName @@ -283,7 +249,7 @@ func (c *GatewayProcessor) serviceConfigs() map[string]fgw.ServiceConfig { Endpoints: c.calculateEndpoints(svc, svcInfo.svcPortName.Port), } - for _, enricher := range enrichers { + for _, enricher := range c.getServicePolicyEnrichers(svc) { enricher.Enrich(svcPortName, svcCfg) } @@ -303,8 +269,8 @@ func (c *GatewayProcessor) calculateEndpoints(svc *corev1.Service, port *int32) } func (c *GatewayProcessor) upstreamsByEndpoints(svc *corev1.Service, port *int32) map[string]fgw.Endpoint { - eps, err := c.cache.informers.GetListers().Endpoints.Endpoints(svc.Namespace).Get(svc.Name) - if err != nil { + eps := &corev1.Endpoints{} + if err := c.cache.client.Get(context.TODO(), client.ObjectKeyFromObject(svc), eps); err != nil { log.Error().Msgf("Failed to get Endpoints of Service %s/%s: %s", svc.Namespace, svc.Name, err) return nil } @@ -343,13 +309,13 @@ func (c *GatewayProcessor) upstreamsByEndpointSlices(svc *corev1.Service, port * return nil } - endpointSliceList, err := c.cache.informers.GetListers().EndpointSlice.EndpointSlices(svc.Namespace).List(selector) - if err != nil { + endpointSliceList := &discoveryv1.EndpointSliceList{} + if err := c.cache.client.List(context.TODO(), endpointSliceList, client.MatchingLabelsSelector{Selector: selector}); err != nil { log.Error().Msgf("Failed to list EndpointSlice of Service %s/%s: %s", svc.Namespace, svc.Name, err) return nil } - if len(endpointSliceList) == 0 { + if len(endpointSliceList.Items) == 0 { return nil } @@ -443,16 +409,21 @@ func (c *GatewayProcessor) chains() fgw.Chains { } func (c *GatewayProcessor) backendRefToServicePortName(referer client.Object, ref gwv1.BackendObjectReference) *fgw.ServicePortName { - if !isValidBackendRefToGroupKindOfService(ref) { + if !gwutils.IsValidBackendRefToGroupKindOfService(ref) { log.Error().Msgf("Unsupported backend group %s and kind %s for service", *ref.Group, *ref.Kind) return nil } + if ref.Port == nil { + log.Warn().Msgf("Port is not specified in the backend reference %s/%s when the referent is a Kubernetes Service", gwutils.NamespaceDerefOr(ref.Namespace, referer.GetNamespace()), ref.Name) + return nil + } + + gvk := referer.GetObjectKind().GroupVersionKind() if ref.Namespace != nil && string(*ref.Namespace) != referer.GetNamespace() && !gwutils.ValidCrossNamespaceRef( - c.referenceGrants, gwtypes.CrossNamespaceFrom{ - Group: referer.GetObjectKind().GroupVersionKind().Group, - Kind: referer.GetObjectKind().GroupVersionKind().Kind, + Group: gvk.Group, + Kind: gvk.Kind, Namespace: referer.GetNamespace(), }, gwtypes.CrossNamespaceTo{ @@ -461,33 +432,34 @@ func (c *GatewayProcessor) backendRefToServicePortName(referer client.Object, re Namespace: string(*ref.Namespace), Name: string(ref.Name), }, + gwutils.GetServiceRefGrants(c.cache.client), ) { log.Error().Msgf("Cross-namespace reference from %s.%s %s/%s to %s.%s %s/%s is not allowed", - referer.GetObjectKind().GroupVersionKind().Kind, referer.GetObjectKind().GroupVersionKind().Group, referer.GetNamespace(), referer.GetName(), + gvk.Kind, gvk.Group, referer.GetNamespace(), referer.GetName(), string(*ref.Kind), string(*ref.Group), string(*ref.Namespace), ref.Name) return nil } return &fgw.ServicePortName{ NamespacedName: types.NamespacedName{ - Namespace: gwutils.Namespace(ref.Namespace, referer.GetNamespace()), + Namespace: gwutils.NamespaceDerefOr(ref.Namespace, referer.GetNamespace()), Name: string(ref.Name), }, - Port: pointer.Int32(int32(*ref.Port)), + Port: ptr.To(int32(*ref.Port)), } } func (c *GatewayProcessor) targetRefToServicePortName(referer client.Object, ref gwv1alpha2.NamespacedPolicyTargetReference, port int32) *fgw.ServicePortName { - if !isValidTargetRefToGroupKindOfService(ref) { + if !gwutils.IsValidTargetRefToGroupKindOfService(ref) { log.Error().Msgf("Unsupported target group %s and kind %s for service", ref.Group, ref.Kind) return nil } + gvk := referer.GetObjectKind().GroupVersionKind() if ref.Namespace != nil && string(*ref.Namespace) != referer.GetNamespace() && !gwutils.ValidCrossNamespaceRef( - c.referenceGrants, gwtypes.CrossNamespaceFrom{ - Group: referer.GetObjectKind().GroupVersionKind().Group, - Kind: referer.GetObjectKind().GroupVersionKind().Kind, + Group: gvk.Group, + Kind: gvk.Kind, Namespace: referer.GetNamespace(), }, gwtypes.CrossNamespaceTo{ @@ -496,16 +468,17 @@ func (c *GatewayProcessor) targetRefToServicePortName(referer client.Object, ref Namespace: string(*ref.Namespace), Name: string(ref.Name), }, + gwutils.GetServiceRefGrants(c.cache.client), ) { log.Error().Msgf("Cross-namespace reference from %s.%s %s/%s to %s.%s %s/%s is not allowed", - referer.GetObjectKind().GroupVersionKind().Kind, referer.GetObjectKind().GroupVersionKind().Group, referer.GetNamespace(), referer.GetName(), + gvk.Kind, gvk.Group, referer.GetNamespace(), referer.GetName(), string(ref.Kind), string(ref.Group), string(*ref.Namespace), ref.Name) return nil } return &fgw.ServicePortName{ NamespacedName: types.NamespacedName{ - Namespace: gwutils.Namespace(ref.Namespace, referer.GetNamespace()), + Namespace: gwutils.NamespaceDerefOr(ref.Namespace, referer.GetNamespace()), Name: string(ref.Name), }, Port: pointer.Int32(port), @@ -608,13 +581,12 @@ func (c *GatewayProcessor) toFSMGRPCRouteFilter(referer client.Object, filter gw } func (c *GatewayProcessor) secretRefToSecret(referer client.Object, ref gwv1.SecretObjectReference) (*corev1.Secret, error) { - if !isValidRefToGroupKindOfSecret(ref) { + if !gwutils.IsValidRefToGroupKindOfSecret(ref) { return nil, fmt.Errorf("unsupported group %s and kind %s for secret", *ref.Group, *ref.Kind) } // If the secret is in a different namespace than the referer, check ReferenceGrants if ref.Namespace != nil && string(*ref.Namespace) != referer.GetNamespace() && !gwutils.ValidCrossNamespaceRef( - c.referenceGrants, gwtypes.CrossNamespaceFrom{ Group: referer.GetObjectKind().GroupVersionKind().Group, Kind: referer.GetObjectKind().GroupVersionKind().Kind, @@ -626,6 +598,7 @@ func (c *GatewayProcessor) secretRefToSecret(referer client.Object, ref gwv1.Sec Namespace: string(*ref.Namespace), Name: string(ref.Name), }, + gwutils.GetSecretRefGrants(c.cache.client), ) { return nil, fmt.Errorf("cross-namespace secert reference from %s.%s %s/%s to %s.%s %s/%s is not allowed", referer.GetObjectKind().GroupVersionKind().Kind, referer.GetObjectKind().GroupVersionKind().Group, referer.GetNamespace(), referer.GetName(), @@ -633,19 +606,18 @@ func (c *GatewayProcessor) secretRefToSecret(referer client.Object, ref gwv1.Sec } return c.cache.getSecretFromCache(client.ObjectKey{ - Namespace: gwutils.Namespace(ref.Namespace, referer.GetNamespace()), + Namespace: gwutils.NamespaceDerefOr(ref.Namespace, referer.GetNamespace()), Name: string(ref.Name), }) } func (c *GatewayProcessor) objectRefToCACertificate(referer client.Object, ref gwv1.ObjectReference) ([]byte, error) { - if !isValidRefToGroupKindOfCA(ref) { + if !gwutils.IsValidRefToGroupKindOfCA(ref) { return nil, fmt.Errorf("unsupported group %s and kind %s for secret", ref.Group, ref.Kind) } // If the secret is in a different namespace than the referer, check ReferenceGrants if ref.Namespace != nil && string(*ref.Namespace) != referer.GetNamespace() && !gwutils.ValidCrossNamespaceRef( - c.referenceGrants, gwtypes.CrossNamespaceFrom{ Group: referer.GetObjectKind().GroupVersionKind().Group, Kind: referer.GetObjectKind().GroupVersionKind().Kind, @@ -657,6 +629,7 @@ func (c *GatewayProcessor) objectRefToCACertificate(referer client.Object, ref g Namespace: string(*ref.Namespace), Name: string(ref.Name), }, + gwutils.GetCARefGrants(c.cache.client), ) { return nil, fmt.Errorf("cross-namespace secert reference from %s.%s %s/%s to %s.%s %s/%s is not allowed", referer.GetObjectKind().GroupVersionKind().Kind, referer.GetObjectKind().GroupVersionKind().Group, referer.GetNamespace(), referer.GetName(), @@ -668,7 +641,7 @@ func (c *GatewayProcessor) objectRefToCACertificate(referer client.Object, ref g switch ref.Kind { case constants.KubernetesSecretKind: secret, err := c.cache.getSecretFromCache(client.ObjectKey{ - Namespace: gwutils.Namespace(ref.Namespace, referer.GetNamespace()), + Namespace: gwutils.NamespaceDerefOr(ref.Namespace, referer.GetNamespace()), Name: string(ref.Name), }) if err != nil { @@ -681,7 +654,7 @@ func (c *GatewayProcessor) objectRefToCACertificate(referer client.Object, ref g } case constants.KubernetesConfigMapKind: cm, err := c.cache.getConfigMapFromCache(client.ObjectKey{ - Namespace: gwutils.Namespace(ref.Namespace, referer.GetNamespace()), + Namespace: gwutils.NamespaceDerefOr(ref.Namespace, referer.GetNamespace()), Name: string(ref.Name), }) if err != nil { @@ -695,7 +668,7 @@ func (c *GatewayProcessor) objectRefToCACertificate(referer client.Object, ref g } if len(ca) == 0 { - return nil, fmt.Errorf("no CA certificate found in %s %s/%s", ref.Kind, gwutils.Namespace(ref.Namespace, referer.GetNamespace()), ref.Name) + return nil, fmt.Errorf("no CA certificate found in %s %s/%s", ref.Kind, gwutils.NamespaceDerefOr(ref.Namespace, referer.GetNamespace()), ref.Name) } return ca, nil diff --git a/pkg/gateway/cache/tcproute.go b/pkg/gateway/cache/tcproute.go index 4bbe86b81..48c20615a 100644 --- a/pkg/gateway/cache/tcproute.go +++ b/pkg/gateway/cache/tcproute.go @@ -1,6 +1,15 @@ package cache import ( + "context" + + "k8s.io/apimachinery/pkg/fields" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/flomesh-io/fsm/pkg/gateway/status/route" + + "github.com/flomesh-io/fsm/pkg/constants" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" gwv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" @@ -8,13 +17,38 @@ import ( gwutils "github.com/flomesh-io/fsm/pkg/gateway/utils" ) +func (c *GatewayProcessor) processTCPRoutes() { + list := &gwv1alpha2.TCPRouteList{} + err := c.cache.client.List(context.Background(), list, &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(constants.GatewayTCPRouteIndex, client.ObjectKeyFromObject(c.gateway).String()), + }) + if err != nil { + log.Error().Msgf("Failed to list TCPRoutes: %v", err) + return + } + + for _, tcpRoute := range gwutils.SortResources(gwutils.ToSlicePtr(list.Items)) { + c.processTCPRoute(tcpRoute) + } +} + func (c *GatewayProcessor) processTCPRoute(tcpRoute *gwv1alpha2.TCPRoute) { - for _, ref := range tcpRoute.Spec.ParentRefs { - if !gwutils.IsRefToGateway(ref, gwutils.ObjectKey(c.gateway)) { + rsh := route.NewRouteStatusUpdate( + tcpRoute, + &tcpRoute.ObjectMeta, + &tcpRoute.TypeMeta, + nil, + gwutils.ToSlicePtr(tcpRoute.Status.Parents), + ) + + for _, parentRef := range tcpRoute.Spec.ParentRefs { + if !gwutils.IsRefToGateway(parentRef, client.ObjectKeyFromObject(c.gateway)) { continue } - allowedListeners, _ := gwutils.GetAllowedListeners(c.getNamespaceLister(), c.gateway, ref, gwutils.ToRouteContext(tcpRoute), c.validListeners) + h := rsh.StatusUpdateFor(parentRef) + + allowedListeners := gwutils.GetAllowedListeners(c.cache.client, c.gateway, h) if len(allowedListeners) == 0 { continue } diff --git a/pkg/gateway/cache/tlsroute.go b/pkg/gateway/cache/tlsroute.go index dfadddadf..31ca63a04 100644 --- a/pkg/gateway/cache/tlsroute.go +++ b/pkg/gateway/cache/tlsroute.go @@ -1,6 +1,15 @@ package cache import ( + "context" + + "k8s.io/apimachinery/pkg/fields" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/flomesh-io/fsm/pkg/gateway/status/route" + + "github.com/flomesh-io/fsm/pkg/constants" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" gwv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" @@ -8,13 +17,38 @@ import ( gwutils "github.com/flomesh-io/fsm/pkg/gateway/utils" ) +func (c *GatewayProcessor) processTLSRoutes() { + list := &gwv1alpha2.TLSRouteList{} + err := c.cache.client.List(context.Background(), list, &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(constants.GatewayTLSRouteIndex, client.ObjectKeyFromObject(c.gateway).String()), + }) + if err != nil { + log.Error().Msgf("Failed to list TLSRoutes: %v", err) + return + } + + for _, tlsRoute := range gwutils.SortResources(gwutils.ToSlicePtr(list.Items)) { + c.processTLSRoute(tlsRoute) + } +} + func (c *GatewayProcessor) processTLSRoute(tlsRoute *gwv1alpha2.TLSRoute) { - for _, ref := range tlsRoute.Spec.ParentRefs { - if !gwutils.IsRefToGateway(ref, gwutils.ObjectKey(c.gateway)) { + rsh := route.NewRouteStatusUpdate( + tlsRoute, + &tlsRoute.ObjectMeta, + &tlsRoute.TypeMeta, + tlsRoute.Spec.Hostnames, + gwutils.ToSlicePtr(tlsRoute.Status.Parents), + ) + + for _, parentRef := range tlsRoute.Spec.ParentRefs { + if !gwutils.IsRefToGateway(parentRef, client.ObjectKeyFromObject(c.gateway)) { continue } - allowedListeners, _ := gwutils.GetAllowedListeners(c.getNamespaceLister(), c.gateway, ref, gwutils.ToRouteContext(tlsRoute), c.validListeners) + h := rsh.StatusUpdateFor(parentRef) + + allowedListeners := gwutils.GetAllowedListeners(c.cache.client, c.gateway, h) if len(allowedListeners) == 0 { continue } diff --git a/pkg/gateway/cache/triggers.go b/pkg/gateway/cache/triggers.go new file mode 100644 index 000000000..d2436c6ac --- /dev/null +++ b/pkg/gateway/cache/triggers.go @@ -0,0 +1,226 @@ +package cache + +import ( + "context" + + "k8s.io/apimachinery/pkg/types" + + "k8s.io/apimachinery/pkg/fields" + "sigs.k8s.io/controller-runtime/pkg/client" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" + gwv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + + gwpav1alpha1 "github.com/flomesh-io/fsm/pkg/apis/policyattachment/v1alpha1" + "github.com/flomesh-io/fsm/pkg/constants" + gwutils "github.com/flomesh-io/fsm/pkg/gateway/utils" +) + +/** + * This file contains the trigger functions for the GatewayCache. + * These functions are used to roughly checking if the resources is referred to by another resource, + * no need to check ReferenceGrants here, over reaction to check ReferenceGrants will cause performance issue + * will compute with ReferenceGrants when generating configuration + */ + +// no need to check ReferenceGrant here +// isRoutableService checks if the service is referred by HTTPRoute/GRPCRoute/TCPRoute/UDPRoute/TLSRoute backendRefs +func (c *GatewayCache) isRoutableService(service client.ObjectKey) bool { + for _, fn := range []func(client.ObjectKey) bool{ + c.isRoutableHTTPService, + c.isRoutableGRPCService, + c.isRoutableTLSService, + c.isRoutableTCPService, + c.isRoutableUDPService, + } { + if fn(service) { + return true + } + } + + return false +} + +// no need to check ReferenceGrant here +func (c *GatewayCache) isRoutableHTTPService(service client.ObjectKey) bool { + list := &gwv1.HTTPRouteList{} + if err := c.client.List(context.Background(), list, &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(constants.BackendHTTPRouteIndex, service.String()), + }); err != nil { + log.Error().Msgf("Failed to list HTTPRoutes: %v", err) + return false + } + + return len(list.Items) > 0 +} + +// no need to check ReferenceGrant here +func (c *GatewayCache) isRoutableGRPCService(service client.ObjectKey) bool { + list := &gwv1.GRPCRouteList{} + if err := c.client.List(context.Background(), list, &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(constants.BackendGRPCRouteIndex, service.String()), + }); err != nil { + log.Error().Msgf("Failed to list GRPCRoutes: %v", err) + return false + } + + return len(list.Items) > 0 +} + +// no need to check ReferenceGrant here +func (c *GatewayCache) isRoutableTLSService(service client.ObjectKey) bool { + list := &gwv1alpha2.TLSRouteList{} + if err := c.client.List(context.Background(), list, &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(constants.BackendTLSRouteIndex, service.String()), + }); err != nil { + log.Error().Msgf("Failed to list TLSRoutes: %v", err) + return false + } + + return len(list.Items) > 0 +} + +// no need to check ReferenceGrant here +func (c *GatewayCache) isRoutableTCPService(service client.ObjectKey) bool { + list := &gwv1alpha2.TCPRouteList{} + if err := c.client.List(context.Background(), list, &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(constants.BackendTCPRouteIndex, service.String()), + }); err != nil { + log.Error().Msgf("Failed to list TCPRoutes: %v", err) + return false + } + + return len(list.Items) > 0 +} + +// no need to check ReferenceGrant here +func (c *GatewayCache) isRoutableUDPService(service client.ObjectKey) bool { + list := &gwv1alpha2.UDPRouteList{} + if err := c.client.List(context.Background(), list, &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(constants.BackendUDPRouteIndex, service.String()), + }); err != nil { + log.Error().Msgf("Failed to list UDPRoutes: %v", err) + return false + } + + return len(list.Items) > 0 +} + +// no need to check ReferenceGrant here +// isEffectiveRoute checks if the route has reference to active Gateway, +func (c *GatewayCache) isEffectiveRoute(parentRefs []gwv1.ParentReference) bool { + gateways := gwutils.GetActiveGateways(c.client) + + if len(gateways) == 0 { + return false + } + + for _, parentRef := range parentRefs { + for _, gw := range gateways { + if gwutils.IsRefToGateway(parentRef, client.ObjectKeyFromObject(gw)) { + return true + } + } + } + + return false +} + +// no need to check ReferenceGrant here +// isEffectiveTargetRef checks if the targetRef is effective, +// it's used to check ONLY policy attachments those are targeting Gateway or HTTPRoute/GRPCRoute resources +func (c *GatewayCache) isEffectiveTargetRef(policy client.Object, targetRef gwv1alpha2.NamespacedPolicyTargetReference) bool { + if targetRef.Group != constants.GatewayAPIGroup { + return false + } + + //referenceGrants := c.getReferenceGrantsFromCache() + key := types.NamespacedName{ + Namespace: gwutils.NamespaceDerefOr(targetRef.Namespace, policy.GetNamespace()), + Name: string(targetRef.Name), + } + + switch targetRef.Kind { + case constants.GatewayAPIGatewayKind: + gw := &gwv1.Gateway{} + if err := c.client.Get(context.Background(), key, gw); err != nil { + log.Error().Msgf("Failed to get Gateway: %v", err) + return false + } + + return gwutils.IsActiveGateway(gw) + case constants.GatewayAPIHTTPRouteKind: + route := &gwv1.HTTPRoute{} + if err := c.client.Get(context.Background(), key, route); err != nil { + log.Error().Msgf("Failed to get HTTPRoute: %v", err) + return false + } + + return true + case constants.GatewayAPIGRPCRouteKind: + route := &gwv1.GRPCRoute{} + if err := c.client.Get(context.Background(), key, route); err != nil { + log.Error().Msgf("Failed to get GRPCRoute: %v", err) + return false + } + + return true + } + + return false +} + +// no need to check ReferenceGrant here +// isRoutableTargetService checks if the targetRef is a valid kind of service, +// routable means it's a service that is referred by HTTPRoute/GRPCRoute/TCPRoute/UDPRoute/TLSRoute backendRefs +func (c *GatewayCache) isRoutableTargetService(owner client.Object, targetRef gwv1alpha2.NamespacedPolicyTargetReference) bool { + if (targetRef.Group == constants.KubernetesCoreGroup && targetRef.Kind == constants.KubernetesServiceKind) || + (targetRef.Group == constants.FlomeshMCSAPIGroup && targetRef.Kind == constants.FlomeshAPIServiceImportKind) { + return c.isRoutableService(client.ObjectKey{ + Namespace: gwutils.NamespaceDerefOr(targetRef.Namespace, owner.GetNamespace()), + Name: string(targetRef.Name), + }) + } + + return false +} + +// no need to check ReferenceGrant here +// isSecretReferred checks if the secret is referred by Gateway or UpstreamTLSPolicy +func (c *GatewayCache) isSecretReferred(secret client.ObjectKey) bool { + list := &gwv1.GatewayList{} + if err := c.client.List(context.Background(), list, &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(constants.SecretGatewayIndex, secret.String()), + }); err != nil { + log.Error().Msgf("Failed to list Gateways: %v", err) + return false + } + + if len(list.Items) > 0 { + return true + } + + policies := &gwpav1alpha1.UpstreamTLSPolicyList{} + if err := c.client.List(context.Background(), policies, &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(constants.SecretUpstreamTLSPolicyIndex, secret.String()), + }); err != nil { + log.Error().Msgf("Failed to list UpstreamTLSPolicyList: %v", err) + return false + } + + return len(list.Items) > 0 +} + +// no need to check ReferenceGrant here +// isConfigMapReferred checks if the configMap is referred by Gateway to store the configuration of gateway or CA certificates +func (c *GatewayCache) isConfigMapReferred(cm client.ObjectKey) bool { + //ctx := context.TODO() + list := &gwv1.GatewayList{} + if err := c.client.List(context.Background(), list, &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(constants.ConfigMapGatewayIndex, cm.String()), + }); err != nil { + log.Error().Msgf("Failed to list Gateways: %v", err) + return false + } + + return len(list.Items) > 0 +} diff --git a/pkg/gateway/cache/types.go b/pkg/gateway/cache/types.go index 8dabb0f4f..d4b0a8e6a 100644 --- a/pkg/gateway/cache/types.go +++ b/pkg/gateway/cache/types.go @@ -28,9 +28,7 @@ package cache import ( corev1 "k8s.io/api/core/v1" - gwpav1alpha1 "github.com/flomesh-io/fsm/pkg/apis/policyattachment/v1alpha1" "github.com/flomesh-io/fsm/pkg/gateway/fgw" - gwpkg "github.com/flomesh-io/fsm/pkg/gateway/types" "github.com/flomesh-io/fsm/pkg/logger" ) @@ -49,7 +47,6 @@ type Cache interface { type serviceContext struct { svcPortName fgw.ServicePortName - //filters []routecfg.Filter } type endpointContext struct { @@ -59,24 +56,6 @@ type endpointContext struct { type calculateEndpointsFunc func(svc *corev1.Service, port *int32) map[string]fgw.Endpoint -type globalPolicyAttachments struct { - rateLimits map[gwpkg.PolicyMatchType][]gwpav1alpha1.RateLimitPolicy - accessControls map[gwpkg.PolicyMatchType][]gwpav1alpha1.AccessControlPolicy - faultInjections map[gwpkg.PolicyMatchType][]gwpav1alpha1.FaultInjectionPolicy -} - -type routePolicies struct { - hostnamesRateLimits []gwpav1alpha1.RateLimitPolicy - httpRouteRateLimits []gwpav1alpha1.RateLimitPolicy - grpcRouteRateLimits []gwpav1alpha1.RateLimitPolicy - hostnamesAccessControls []gwpav1alpha1.AccessControlPolicy - httpRouteAccessControls []gwpav1alpha1.AccessControlPolicy - grpcRouteAccessControls []gwpav1alpha1.AccessControlPolicy - hostnamesFaultInjections []gwpav1alpha1.FaultInjectionPolicy - httpRouteFaultInjections []gwpav1alpha1.FaultInjectionPolicy - grpcRouteFaultInjections []gwpav1alpha1.FaultInjectionPolicy -} - var ( log = logger.New("fsm-gateway/cache") ) diff --git a/pkg/gateway/cache/udproute.go b/pkg/gateway/cache/udproute.go index ad58bd0ee..94e2d1cd3 100644 --- a/pkg/gateway/cache/udproute.go +++ b/pkg/gateway/cache/udproute.go @@ -1,6 +1,15 @@ package cache import ( + "context" + + "k8s.io/apimachinery/pkg/fields" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/flomesh-io/fsm/pkg/gateway/status/route" + + "github.com/flomesh-io/fsm/pkg/constants" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" gwv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" @@ -8,13 +17,38 @@ import ( gwutils "github.com/flomesh-io/fsm/pkg/gateway/utils" ) +func (c *GatewayProcessor) processUDPRoutes() { + list := &gwv1alpha2.UDPRouteList{} + err := c.cache.client.List(context.Background(), list, &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(constants.GatewayUDPRouteIndex, client.ObjectKeyFromObject(c.gateway).String()), + }) + if err != nil { + log.Error().Msgf("Failed to list UDPRoutes: %v", err) + return + } + + for _, udpRoute := range gwutils.SortResources(gwutils.ToSlicePtr(list.Items)) { + c.processUDPRoute(udpRoute) + } +} + func (c *GatewayProcessor) processUDPRoute(udpRoute *gwv1alpha2.UDPRoute) { - for _, ref := range udpRoute.Spec.ParentRefs { - if !gwutils.IsRefToGateway(ref, gwutils.ObjectKey(c.gateway)) { + rsh := route.NewRouteStatusUpdate( + udpRoute, + &udpRoute.ObjectMeta, + &udpRoute.TypeMeta, + nil, + gwutils.ToSlicePtr(udpRoute.Status.Parents), + ) + + for _, parentRef := range udpRoute.Spec.ParentRefs { + if !gwutils.IsRefToGateway(parentRef, client.ObjectKeyFromObject(c.gateway)) { continue } - allowedListeners, _ := gwutils.GetAllowedListeners(c.getNamespaceLister(), c.gateway, ref, gwutils.ToRouteContext(udpRoute), c.validListeners) + h := rsh.StatusUpdateFor(parentRef) + + allowedListeners := gwutils.GetAllowedListeners(c.cache.client, c.gateway, h) if len(allowedListeners) == 0 { continue } diff --git a/pkg/gateway/cache/utils.go b/pkg/gateway/cache/utils.go index 1eda30f57..d1b4b6fa7 100644 --- a/pkg/gateway/cache/utils.go +++ b/pkg/gateway/cache/utils.go @@ -15,7 +15,6 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/pointer" gwv1 "sigs.k8s.io/gateway-api/apis/v1" - gwv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" "github.com/flomesh-io/fsm/pkg/constants" "github.com/flomesh-io/fsm/pkg/gateway/fgw" @@ -366,12 +365,13 @@ func getServicePort(svc *corev1.Service, port *int32) (corev1.ServicePort, error return corev1.ServicePort{}, fmt.Errorf("no matching port for Service %s and port %d", svc.Name, port) } -func filterEndpointSliceList(endpointSliceList []*discoveryv1.EndpointSlice, port corev1.ServicePort) []*discoveryv1.EndpointSlice { - filtered := make([]*discoveryv1.EndpointSlice, 0, len(endpointSliceList)) +func filterEndpointSliceList(endpointSliceList *discoveryv1.EndpointSliceList, port corev1.ServicePort) []*discoveryv1.EndpointSlice { + filtered := make([]*discoveryv1.EndpointSlice, 0, len(endpointSliceList.Items)) - for _, endpointSlice := range endpointSliceList { - if !ignoreEndpointSlice(endpointSlice, port) { - filtered = append(filtered, endpointSlice) + for _, endpointSlice := range endpointSliceList.Items { + endpointSlice := endpointSlice + if !ignoreEndpointSlice(&endpointSlice, port) { + filtered = append(filtered, &endpointSlice) } } @@ -510,68 +510,6 @@ func passthroughTarget(ref gwv1.BackendRef) *string { return nil } -func isValidRefToGroupKindOfSecret(ref gwv1.SecretObjectReference) bool { - if ref.Group == nil { - return false - } - - if ref.Kind == nil { - return false - } - - if string(*ref.Group) == constants.KubernetesCoreGroup && string(*ref.Kind) == constants.KubernetesSecretKind { - return true - } - - return false -} - -func isValidRefToGroupKindOfConfigMap(ref gwv1.ObjectReference) bool { - if ref.Group == corev1.GroupName && ref.Kind == constants.KubernetesConfigMapKind { - return true - } - - return false -} - -func isValidRefToGroupKindOfCA(ref gwv1.ObjectReference) bool { - if ref.Group != corev1.GroupName { - return false - } - - if ref.Kind == constants.KubernetesSecretKind || ref.Kind == constants.KubernetesConfigMapKind { - return true - } - - return false -} - -func isValidBackendRefToGroupKindOfService(ref gwv1.BackendObjectReference) bool { - if ref.Group == nil { - return false - } - - if ref.Kind == nil { - return false - } - - if (string(*ref.Kind) == constants.KubernetesServiceKind && string(*ref.Group) == constants.KubernetesCoreGroup) || - (string(*ref.Kind) == constants.FlomeshAPIServiceImportKind && string(*ref.Group) == constants.FlomeshMCSAPIGroup) { - return true - } - - return false -} - -func isValidTargetRefToGroupKindOfService(ref gwv1alpha2.NamespacedPolicyTargetReference) bool { - if (ref.Kind == constants.KubernetesServiceKind && ref.Group == constants.KubernetesCoreGroup) || - (ref.Kind == constants.FlomeshAPIServiceImportKind && ref.Group == constants.FlomeshMCSAPIGroup) { - return true - } - - return false -} - func toFGWEndpoints(endpointSet map[endpointContext]struct{}) map[string]fgw.Endpoint { endpoints := make(map[string]fgw.Endpoint) for ep := range endpointSet { diff --git a/pkg/gateway/client.go b/pkg/gateway/client.go index dae448680..a75d94d2d 100644 --- a/pkg/gateway/client.go +++ b/pkg/gateway/client.go @@ -3,6 +3,9 @@ package gateway import ( "context" + gwtypes "github.com/flomesh-io/fsm/pkg/gateway/types" + "github.com/flomesh-io/fsm/pkg/version" + gwpav1alpha1 "github.com/flomesh-io/fsm/pkg/apis/policyattachment/v1alpha1" "github.com/google/go-cmp/cmp" @@ -11,8 +14,8 @@ import ( discoveryv1 "k8s.io/api/discovery/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" k8scache "k8s.io/client-go/tools/cache" + crClient "sigs.k8s.io/controller-runtime/pkg/client" gwv1 "sigs.k8s.io/gateway-api/apis/v1" gwv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" @@ -22,34 +25,37 @@ import ( "github.com/flomesh-io/fsm/pkg/announcements" mcsv1alpha1 "github.com/flomesh-io/fsm/pkg/apis/multicluster/v1alpha1" - "github.com/flomesh-io/fsm/pkg/configurator" + cctx "github.com/flomesh-io/fsm/pkg/context" "github.com/flomesh-io/fsm/pkg/gateway/cache" "github.com/flomesh-io/fsm/pkg/gateway/repo" "github.com/flomesh-io/fsm/pkg/k8s" "github.com/flomesh-io/fsm/pkg/k8s/events" - "github.com/flomesh-io/fsm/pkg/k8s/informers" fsminformers "github.com/flomesh-io/fsm/pkg/k8s/informers" "github.com/flomesh-io/fsm/pkg/logger" - "github.com/flomesh-io/fsm/pkg/messaging" ) var ( log = logger.New("controller-gatewayapi") ) -// NewGatewayAPIController returns a gateway.Controller interface related to functionality provided by the resources in the plugin.flomesh.io API group -func NewGatewayAPIController(informerCollection *fsminformers.InformerCollection, kubeClient kubernetes.Interface, gatewayAPIClient gatewayApiClientset.Interface, msgBroker *messaging.Broker, cfg configurator.Configurator, meshName, fsmVersion string) Controller { - return newClient(informerCollection, kubeClient, gatewayAPIClient, msgBroker, cfg, meshName, fsmVersion) +// NewGatewayAPIController returns a gateway.Controller interface related to functionality provided by the resources in the gateway.flomesh.io API group +func NewGatewayAPIController(ctx *cctx.ControllerContext) gwtypes.Controller { + return newClient(ctx) } -func newClient(informerCollection *informers.InformerCollection, kubeClient kubernetes.Interface, gatewayAPIClient gatewayApiClientset.Interface, msgBroker *messaging.Broker, cfg configurator.Configurator, meshName, fsmVersion string) *client { +func newClient(ctx *cctx.ControllerContext) *client { + gatewayAPIClient, err := gatewayApiClientset.NewForConfig(ctx.KubeConfig) + if err != nil { + panic(err) + } + fsmGatewayClass := &gwv1.GatewayClass{ ObjectMeta: metav1.ObjectMeta{ Name: constants.FSMGatewayClassName, Labels: map[string]string{ constants.FSMAppNameLabelKey: constants.FSMAppNameLabelValue, - constants.FSMAppInstanceLabelKey: meshName, - constants.FSMAppVersionLabelKey: fsmVersion, + constants.FSMAppInstanceLabelKey: ctx.MeshName, + constants.FSMAppVersionLabelKey: ctx.FSMVersion, constants.AppLabel: constants.FSMGatewayName, }, }, @@ -65,47 +71,65 @@ func newClient(informerCollection *informers.InformerCollection, kubeClient kube } c := &client{ - informers: informerCollection, - kubeClient: kubeClient, - msgBroker: msgBroker, - cfg: cfg, - cache: cache.NewGatewayCache(informerCollection, kubeClient, cfg), + msgBroker: ctx.MsgBroker, + cfg: ctx.Configurator, + cache: cache.NewGatewayCache(ctx), } // Initialize informers - for _, informerKey := range []fsminformers.InformerKey{ - fsminformers.InformerKeyService, - fsminformers.InformerKeyServiceImport, - fsminformers.InformerKeyEndpoints, - fsminformers.InformerKeyEndpointSlices, - fsminformers.InformerKeySecret, - fsminformers.InformerKeyConfigMap, - fsminformers.InformerKeyGatewayAPIGatewayClass, - fsminformers.InformerKeyGatewayAPIGateway, - fsminformers.InformerKeyGatewayAPIHTTPRoute, - fsminformers.InformerKeyGatewayAPIGRPCRoute, - fsminformers.InformerKeyGatewayAPITLSRoute, - fsminformers.InformerKeyGatewayAPITCPRoute, - fsminformers.InformerKeyGatewayAPIUDPRoute, - fsminformers.InformerKeyGatewayAPIReferenceGrant, - fsminformers.InformerKeyRateLimitPolicy, - fsminformers.InformerKeySessionStickyPolicy, - fsminformers.InformerKeyLoadBalancerPolicy, - fsminformers.InformerKeyCircuitBreakingPolicy, - fsminformers.InformerKeyAccessControlPolicy, - fsminformers.InformerKeyHealthCheckPolicy, - fsminformers.InformerKeyFaultInjectionPolicy, - fsminformers.InformerKeyUpstreamTLSPolicy, - fsminformers.InformerKeyRetryPolicy, - } { + informers := map[fsminformers.InformerKey]crClient.Object{ + fsminformers.InformerKeyService: &corev1.Service{}, + fsminformers.InformerKeyServiceImport: &mcsv1alpha1.ServiceImport{}, + fsminformers.InformerKeyEndpoints: &corev1.Endpoints{}, + fsminformers.InformerKeySecret: &corev1.Secret{}, + fsminformers.InformerKeyConfigMap: &corev1.ConfigMap{}, + fsminformers.InformerKeyGatewayAPIGatewayClass: &gwv1.GatewayClass{}, + fsminformers.InformerKeyGatewayAPIGateway: &gwv1.Gateway{}, + fsminformers.InformerKeyGatewayAPIHTTPRoute: &gwv1.HTTPRoute{}, + fsminformers.InformerKeyGatewayAPIGRPCRoute: &gwv1.GRPCRoute{}, + fsminformers.InformerKeyGatewayAPITLSRoute: &gwv1alpha2.TLSRoute{}, + fsminformers.InformerKeyGatewayAPITCPRoute: &gwv1alpha2.TCPRoute{}, + fsminformers.InformerKeyGatewayAPIUDPRoute: &gwv1alpha2.UDPRoute{}, + fsminformers.InformerKeyGatewayAPIReferenceGrant: &gwv1beta1.ReferenceGrant{}, + fsminformers.InformerKeyRateLimitPolicy: &gwpav1alpha1.RateLimitPolicy{}, + fsminformers.InformerKeySessionStickyPolicy: &gwpav1alpha1.SessionStickyPolicy{}, + fsminformers.InformerKeyLoadBalancerPolicy: &gwpav1alpha1.LoadBalancerPolicy{}, + fsminformers.InformerKeyCircuitBreakingPolicy: &gwpav1alpha1.CircuitBreakingPolicy{}, + fsminformers.InformerKeyAccessControlPolicy: &gwpav1alpha1.AccessControlPolicy{}, + fsminformers.InformerKeyHealthCheckPolicy: &gwpav1alpha1.HealthCheckPolicy{}, + fsminformers.InformerKeyFaultInjectionPolicy: &gwpav1alpha1.FaultInjectionPolicy{}, + fsminformers.InformerKeyUpstreamTLSPolicy: &gwpav1alpha1.UpstreamTLSPolicy{}, + fsminformers.InformerKeyRetryPolicy: &gwpav1alpha1.RetryPolicy{}, + fsminformers.InformerKeyNamespace: &corev1.Namespace{}, + } + + if version.IsEndpointSliceEnabled(ctx.KubeClient) { + informers[fsminformers.InformerKeyEndpointSlices] = &discoveryv1.EndpointSlice{} + } + + for informerKey, resource := range informers { if eventTypes := getEventTypesByInformerKey(informerKey); eventTypes != nil { - c.informers.AddEventHandler(informerKey, c.getEventHandlerFuncs(eventTypes)) + c.informOnResource(ctx, resource, c.getEventHandlerFuncs(eventTypes)) } } return c } +func (c *client) informOnResource(ctx *cctx.ControllerContext, obj crClient.Object, handler k8scache.ResourceEventHandlerFuncs) { + ch := ctx.Manager.GetCache() + + informer, err := ch.GetInformer(context.Background(), obj) + if err != nil { + panic(err) + } + + _, err = informer.AddEventHandler(handler) + if err != nil { + panic(err) + } +} + func (c *client) getEventHandlerFuncs(eventTypes *k8s.EventTypes) k8scache.ResourceEventHandlerFuncs { return k8scache.ResourceEventHandlerFuncs{ AddFunc: c.onAddFunc(eventTypes), diff --git a/pkg/gateway/fgw/types.go b/pkg/gateway/fgw/types.go index ff95cde2c..fb9f02c13 100644 --- a/pkg/gateway/fgw/types.go +++ b/pkg/gateway/fgw/types.go @@ -432,6 +432,13 @@ type HTTPTrafficMatch struct { AccessControlLists *AccessControlLists `json:"AccessControlLists,omitempty"` FaultInjection *FaultInjection `json:"Fault,omitempty"` Filters []Filter `json:"Filters,omitempty" hash:"set"` + Timeouts *HTTPRouteTimeouts `json:"Timeout,omitempty"` +} + +// HTTPRouteTimeouts is the HTTP route timeouts configuration +type HTTPRouteTimeouts struct { + Request *gwv1.Duration `json:"Request,omitempty"` + BackendRequest *gwv1.Duration `json:"BackendRequest,omitempty"` } // GRPCTrafficMatch is the GRPC traffic match configuration diff --git a/pkg/gateway/mock_client_generated.go b/pkg/gateway/mock_client_generated.go index 22a04295b..5325dddd6 100644 --- a/pkg/gateway/mock_client_generated.go +++ b/pkg/gateway/mock_client_generated.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/flomesh-io/fsm/pkg/gateway (interfaces: Controller) +// Source: github.com/flomesh-io/fsm/pkg/gateway/types (interfaces: Controller) // Package gateway is a generated GoMock package. package gateway diff --git a/pkg/gateway/policy/grpc_route_enricher.go b/pkg/gateway/policy/grpc_route_enricher.go index 440245907..df245f9de 100644 --- a/pkg/gateway/policy/grpc_route_enricher.go +++ b/pkg/gateway/policy/grpc_route_enricher.go @@ -1,8 +1,13 @@ package policy import ( + "k8s.io/apimachinery/pkg/fields" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" gwv1 "sigs.k8s.io/gateway-api/apis/v1" + gwutils "github.com/flomesh-io/fsm/pkg/gateway/utils" + "github.com/flomesh-io/fsm/pkg/gateway/policy/utils/accesscontrol" "github.com/flomesh-io/fsm/pkg/gateway/policy/utils/faultinjection" "github.com/flomesh-io/fsm/pkg/gateway/policy/utils/ratelimit" @@ -17,17 +22,24 @@ type GRPCRoutePolicyEnricher interface { // --- -// RateLimitGRPCRouteEnricher is an enricher for rate limit policies at the GRPC route level -type RateLimitGRPCRouteEnricher struct { - Data []gwpav1alpha1.RateLimitPolicy +func NewRateLimitGRPCRouteEnricher(cache cache.Cache, selector fields.Selector) GRPCRoutePolicyEnricher { + return &rateLimitGRPCRouteEnricher{ + data: gwutils.SortResources(gwutils.GetRateLimitsMatchTypeGRPCRoute(cache, selector)), + } } -func (e *RateLimitGRPCRouteEnricher) Enrich(match gwv1.GRPCRouteMatch, matchCfg *fgw.GRPCTrafficMatch) { - if len(e.Data) == 0 { +// rateLimitGRPCRouteEnricher is an enricher for rate limit policies at the GRPC route level +type rateLimitGRPCRouteEnricher struct { + data []client.Object +} + +func (e *rateLimitGRPCRouteEnricher) Enrich(match gwv1.GRPCRouteMatch, matchCfg *fgw.GRPCTrafficMatch) { + if len(e.data) == 0 { return } - for _, rateLimit := range e.Data { + for _, rateLimit := range e.data { + rateLimit := rateLimit.(*gwpav1alpha1.RateLimitPolicy) if len(rateLimit.Spec.GRPCRateLimits) == 0 { continue } @@ -41,17 +53,24 @@ func (e *RateLimitGRPCRouteEnricher) Enrich(match gwv1.GRPCRouteMatch, matchCfg // --- -// AccessControlGRPCRouteEnricher is an enricher for access control policies at the GRPC route level -type AccessControlGRPCRouteEnricher struct { - Data []gwpav1alpha1.AccessControlPolicy +func NewAccessControlGRPCRouteEnricher(cache cache.Cache, selector fields.Selector) GRPCRoutePolicyEnricher { + return &accessControlGRPCRouteEnricher{ + data: gwutils.SortResources(gwutils.GetAccessControlsMatchTypeGRPCRoute(cache, selector)), + } +} + +// accessControlGRPCRouteEnricher is an enricher for access control policies at the GRPC route level +type accessControlGRPCRouteEnricher struct { + data []client.Object } -func (e *AccessControlGRPCRouteEnricher) Enrich(match gwv1.GRPCRouteMatch, matchCfg *fgw.GRPCTrafficMatch) { - if len(e.Data) == 0 { +func (e *accessControlGRPCRouteEnricher) Enrich(match gwv1.GRPCRouteMatch, matchCfg *fgw.GRPCTrafficMatch) { + if len(e.data) == 0 { return } - for _, accessControl := range e.Data { + for _, accessControl := range e.data { + accessControl := accessControl.(*gwpav1alpha1.AccessControlPolicy) if len(accessControl.Spec.GRPCAccessControls) == 0 { continue } @@ -65,17 +84,24 @@ func (e *AccessControlGRPCRouteEnricher) Enrich(match gwv1.GRPCRouteMatch, match // --- -// FaultInjectionGRPCRouteEnricher is an enricher for fault injection policies at the GRPC route level -type FaultInjectionGRPCRouteEnricher struct { - Data []gwpav1alpha1.FaultInjectionPolicy +func NewFaultInjectionGRPCRouteEnricher(cache cache.Cache, selector fields.Selector) GRPCRoutePolicyEnricher { + return &faultInjectionGRPCRouteEnricher{ + data: gwutils.SortResources(gwutils.GetFaultInjectionsMatchTypeGRPCRoute(cache, selector)), + } +} + +// faultInjectionGRPCRouteEnricher is an enricher for fault injection policies at the GRPC route level +type faultInjectionGRPCRouteEnricher struct { + data []client.Object } -func (e *FaultInjectionGRPCRouteEnricher) Enrich(match gwv1.GRPCRouteMatch, matchCfg *fgw.GRPCTrafficMatch) { - if len(e.Data) == 0 { +func (e *faultInjectionGRPCRouteEnricher) Enrich(match gwv1.GRPCRouteMatch, matchCfg *fgw.GRPCTrafficMatch) { + if len(e.data) == 0 { return } - for _, faultInjection := range e.Data { + for _, faultInjection := range e.data { + faultInjection := faultInjection.(*gwpav1alpha1.FaultInjectionPolicy) if len(faultInjection.Spec.GRPCFaultInjections) == 0 { continue } diff --git a/pkg/gateway/policy/hostname_enricher.go b/pkg/gateway/policy/hostname_enricher.go index 05c37a499..a2ef842bc 100644 --- a/pkg/gateway/policy/hostname_enricher.go +++ b/pkg/gateway/policy/hostname_enricher.go @@ -1,11 +1,16 @@ package policy import ( + "k8s.io/apimachinery/pkg/fields" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + gwpav1alpha1 "github.com/flomesh-io/fsm/pkg/apis/policyattachment/v1alpha1" "github.com/flomesh-io/fsm/pkg/gateway/fgw" "github.com/flomesh-io/fsm/pkg/gateway/policy/utils/accesscontrol" "github.com/flomesh-io/fsm/pkg/gateway/policy/utils/faultinjection" "github.com/flomesh-io/fsm/pkg/gateway/policy/utils/ratelimit" + gwutils "github.com/flomesh-io/fsm/pkg/gateway/utils" ) type HostnamePolicyEnricher interface { @@ -14,17 +19,24 @@ type HostnamePolicyEnricher interface { // --- +func NewRateLimitHostnameEnricher(cache cache.Cache, selector fields.Selector) HostnamePolicyEnricher { + return &RateLimitHostnameEnricher{ + data: gwutils.SortResources(gwutils.GetRateLimitsMatchTypeHostname(cache, selector)), + } +} + // RateLimitHostnameEnricher is an enricher for rate limit policies at the hostname level type RateLimitHostnameEnricher struct { - Data []gwpav1alpha1.RateLimitPolicy + data []client.Object } func (e *RateLimitHostnameEnricher) Enrich(hostname string, r fgw.L7RouteRuleSpec) { - if len(e.Data) == 0 { + if len(e.data) == 0 { return } - for _, rateLimit := range e.Data { + for _, rateLimit := range e.data { + rateLimit := rateLimit.(*gwpav1alpha1.RateLimitPolicy) if rl := ratelimit.GetRateLimitIfRouteHostnameMatchesPolicy(hostname, rateLimit); rl != nil && r.GetRateLimit() == nil { r.SetRateLimit(newRateLimitConfig(rl)) break @@ -34,17 +46,24 @@ func (e *RateLimitHostnameEnricher) Enrich(hostname string, r fgw.L7RouteRuleSpe // --- -// AccessControlHostnameEnricher is an enricher for access control policies at the hostname level -type AccessControlHostnameEnricher struct { - Data []gwpav1alpha1.AccessControlPolicy +func NewAccessControlHostnameEnricher(cache cache.Cache, selector fields.Selector) HostnamePolicyEnricher { + return &accessControlHostnameEnricher{ + data: gwutils.SortResources(gwutils.GetAccessControlsMatchTypeHostname(cache, selector)), + } +} + +// accessControlHostnameEnricher is an enricher for access control policies at the hostname level +type accessControlHostnameEnricher struct { + data []client.Object } -func (e *AccessControlHostnameEnricher) Enrich(hostname string, r fgw.L7RouteRuleSpec) { - if len(e.Data) == 0 { +func (e *accessControlHostnameEnricher) Enrich(hostname string, r fgw.L7RouteRuleSpec) { + if len(e.data) == 0 { return } - for _, ac := range e.Data { + for _, ac := range e.data { + ac := ac.(*gwpav1alpha1.AccessControlPolicy) if cfg := accesscontrol.GetAccessControlConfigIfRouteHostnameMatchesPolicy(hostname, ac); cfg != nil && r.GetAccessControlLists() == nil { r.SetAccessControlLists(newAccessControlLists(cfg)) break @@ -54,17 +73,24 @@ func (e *AccessControlHostnameEnricher) Enrich(hostname string, r fgw.L7RouteRul // --- -// FaultInjectionHostnameEnricher is an enricher for fault injection policies at the hostname level -type FaultInjectionHostnameEnricher struct { - Data []gwpav1alpha1.FaultInjectionPolicy +func NewFaultInjectionHostnameEnricher(cache cache.Cache, selector fields.Selector) HostnamePolicyEnricher { + return &faultInjectionHostnameEnricher{ + data: gwutils.SortResources(gwutils.GetFaultInjectionsMatchTypeHostname(cache, selector)), + } +} + +// faultInjectionHostnameEnricher is an enricher for fault injection policies at the hostname level +type faultInjectionHostnameEnricher struct { + data []client.Object } -func (e *FaultInjectionHostnameEnricher) Enrich(hostname string, r fgw.L7RouteRuleSpec) { - if len(e.Data) == 0 { +func (e *faultInjectionHostnameEnricher) Enrich(hostname string, r fgw.L7RouteRuleSpec) { + if len(e.data) == 0 { return } - for _, fj := range e.Data { + for _, fj := range e.data { + fj := fj.(*gwpav1alpha1.FaultInjectionPolicy) if cfg := faultinjection.GetFaultInjectionConfigIfRouteHostnameMatchesPolicy(hostname, fj); cfg != nil && r.GetFaultInjection() == nil { r.SetFaultInjection(newFaultInjection(cfg)) break diff --git a/pkg/gateway/policy/http_route_enricher.go b/pkg/gateway/policy/http_route_enricher.go index 21aaea702..a13d1c2c2 100644 --- a/pkg/gateway/policy/http_route_enricher.go +++ b/pkg/gateway/policy/http_route_enricher.go @@ -1,8 +1,13 @@ package policy import ( + "k8s.io/apimachinery/pkg/fields" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" gwv1 "sigs.k8s.io/gateway-api/apis/v1" + gwutils "github.com/flomesh-io/fsm/pkg/gateway/utils" + "github.com/flomesh-io/fsm/pkg/gateway/policy/utils/accesscontrol" "github.com/flomesh-io/fsm/pkg/gateway/policy/utils/faultinjection" "github.com/flomesh-io/fsm/pkg/gateway/policy/utils/ratelimit" @@ -17,17 +22,25 @@ type HTTPRoutePolicyEnricher interface { // --- -// RateLimitHTTPRouteEnricher is an enricher for rate limit policies at the HTTP route level -type RateLimitHTTPRouteEnricher struct { - Data []gwpav1alpha1.RateLimitPolicy +func NewRateLimitHTTPRouteEnricher(cache cache.Cache, selector fields.Selector) HTTPRoutePolicyEnricher { + return &rateLimitHTTPRouteEnricher{ + data: gwutils.SortResources(gwutils.GetRateLimitsMatchTypeHTTPRoute(cache, selector)), + } +} + +// rateLimitHTTPRouteEnricher is an enricher for rate limit policies at the HTTP route level +type rateLimitHTTPRouteEnricher struct { + data []client.Object } -func (e *RateLimitHTTPRouteEnricher) Enrich(match gwv1.HTTPRouteMatch, matchCfg *fgw.HTTPTrafficMatch) { - if len(e.Data) == 0 { +func (e *rateLimitHTTPRouteEnricher) Enrich(match gwv1.HTTPRouteMatch, matchCfg *fgw.HTTPTrafficMatch) { + if len(e.data) == 0 { return } - for _, rateLimit := range e.Data { + for _, rateLimit := range e.data { + rateLimit := rateLimit.(*gwpav1alpha1.RateLimitPolicy) + if len(rateLimit.Spec.HTTPRateLimits) == 0 { continue } @@ -41,17 +54,25 @@ func (e *RateLimitHTTPRouteEnricher) Enrich(match gwv1.HTTPRouteMatch, matchCfg // --- -// AccessControlHTTPRouteEnricher is an enricher for access control policies at the HTTP route level -type AccessControlHTTPRouteEnricher struct { - Data []gwpav1alpha1.AccessControlPolicy +func NewAccessControlHTTPRouteEnricher(cache cache.Cache, selector fields.Selector) HTTPRoutePolicyEnricher { + return &accessControlHTTPRouteEnricher{ + data: gwutils.SortResources(gwutils.GetAccessControlsMatchTypeHTTPRoute(cache, selector)), + } } -func (e *AccessControlHTTPRouteEnricher) Enrich(match gwv1.HTTPRouteMatch, matchCfg *fgw.HTTPTrafficMatch) { - if len(e.Data) == 0 { +// accessControlHTTPRouteEnricher is an enricher for access control policies at the HTTP route level +type accessControlHTTPRouteEnricher struct { + data []client.Object +} + +func (e *accessControlHTTPRouteEnricher) Enrich(match gwv1.HTTPRouteMatch, matchCfg *fgw.HTTPTrafficMatch) { + if len(e.data) == 0 { return } - for _, accessControl := range e.Data { + for _, accessControl := range e.data { + accessControl := accessControl.(*gwpav1alpha1.AccessControlPolicy) + if len(accessControl.Spec.HTTPAccessControls) == 0 { continue } @@ -65,17 +86,25 @@ func (e *AccessControlHTTPRouteEnricher) Enrich(match gwv1.HTTPRouteMatch, match // --- -// FaultInjectionHTTPRouteEnricher is an enricher for fault injection policies at the HTTP route level -type FaultInjectionHTTPRouteEnricher struct { - Data []gwpav1alpha1.FaultInjectionPolicy +func NewFaultInjectionHTTPRouteEnricher(cache cache.Cache, selector fields.Selector) HTTPRoutePolicyEnricher { + return &faultInjectionHTTPRouteEnricher{ + data: gwutils.SortResources(gwutils.GetFaultInjectionsMatchTypeHTTPRoute(cache, selector)), + } +} + +// faultInjectionHTTPRouteEnricher is an enricher for fault injection policies at the HTTP route level +type faultInjectionHTTPRouteEnricher struct { + data []client.Object } -func (e *FaultInjectionHTTPRouteEnricher) Enrich(match gwv1.HTTPRouteMatch, matchCfg *fgw.HTTPTrafficMatch) { - if len(e.Data) == 0 { +func (e *faultInjectionHTTPRouteEnricher) Enrich(match gwv1.HTTPRouteMatch, matchCfg *fgw.HTTPTrafficMatch) { + if len(e.data) == 0 { return } - for _, faultInjection := range e.Data { + for _, faultInjection := range e.data { + faultInjection := faultInjection.(*gwpav1alpha1.FaultInjectionPolicy) + if len(faultInjection.Spec.HTTPFaultInjections) == 0 { continue } diff --git a/pkg/gateway/policy/port_enrichers.go b/pkg/gateway/policy/port_enrichers.go index 7d3f6b90f..2058c9938 100644 --- a/pkg/gateway/policy/port_enrichers.go +++ b/pkg/gateway/policy/port_enrichers.go @@ -1,15 +1,18 @@ package policy import ( + "k8s.io/apimachinery/pkg/fields" + "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" gwv1 "sigs.k8s.io/gateway-api/apis/v1" "github.com/flomesh-io/fsm/pkg/gateway/policy/utils/accesscontrol" + gwutils "github.com/flomesh-io/fsm/pkg/gateway/utils" + "github.com/flomesh-io/fsm/pkg/gateway/policy/utils/ratelimit" gwpav1alpha1 "github.com/flomesh-io/fsm/pkg/apis/policyattachment/v1alpha1" "github.com/flomesh-io/fsm/pkg/gateway/fgw" - gwutils "github.com/flomesh-io/fsm/pkg/gateway/utils" ) type PortPolicyEnricher interface { @@ -18,24 +21,26 @@ type PortPolicyEnricher interface { // --- -// RateLimitPortEnricher is an enricher for rate limit policies at the port level -type RateLimitPortEnricher struct { - Data []gwpav1alpha1.RateLimitPolicy - ReferenceGrants []client.Object +func NewRateLimitPortEnricher(cache cache.Cache, selector fields.Selector) PortPolicyEnricher { + return &rateLimitPortEnricher{ + data: gwutils.SortResources(gwutils.GetRateLimitsMatchTypePort(cache, selector)), + } } -func (e *RateLimitPortEnricher) Enrich(gw *gwv1.Gateway, port gwv1.PortNumber, listenerCfg *fgw.Listener) { +// rateLimitPortEnricher is an enricher for rate limit policies at the port level +type rateLimitPortEnricher struct { + data []client.Object +} + +func (e *rateLimitPortEnricher) Enrich(gw *gwv1.Gateway, port gwv1.PortNumber, listenerCfg *fgw.Listener) { switch listenerCfg.Protocol { case gwv1.HTTPProtocolType, gwv1.HTTPSProtocolType, gwv1.TLSProtocolType, gwv1.TCPProtocolType: - if len(e.Data) == 0 { + if len(e.data) == 0 { return } - for _, rateLimit := range e.Data { - rateLimit := rateLimit - if !gwutils.IsRefToTarget(e.ReferenceGrants, &rateLimit, rateLimit.Spec.TargetRef, gw) { - continue - } + for _, rateLimit := range e.data { + rateLimit := rateLimit.(*gwpav1alpha1.RateLimitPolicy) if len(rateLimit.Spec.Ports) == 0 { continue @@ -47,41 +52,43 @@ func (e *RateLimitPortEnricher) Enrich(gw *gwv1.Gateway, port gwv1.PortNumber, l } } default: - log.Warn().Msgf("RateLimitPortEnricher: unsupported protocol %s", listenerCfg.Protocol) + log.Warn().Msgf("rateLimitPortEnricher: unsupported protocol %s", listenerCfg.Protocol) } } // --- -// AccessControlPortEnricher is an enricher for access control policies at the port level -type AccessControlPortEnricher struct { - Data []gwpav1alpha1.AccessControlPolicy - ReferenceGrants []client.Object +func NewAccessControlPortEnricher(cache cache.Cache, selector fields.Selector) PortPolicyEnricher { + return &accessControlPortEnricher{ + data: gwutils.SortResources(gwutils.GetAccessControlsMatchTypePort(cache, selector)), + } +} + +// accessControlPortEnricher is an enricher for access control policies at the port level +type accessControlPortEnricher struct { + data []client.Object } -func (e *AccessControlPortEnricher) Enrich(gw *gwv1.Gateway, port gwv1.PortNumber, listenerCfg *fgw.Listener) { +func (e *accessControlPortEnricher) Enrich(gw *gwv1.Gateway, port gwv1.PortNumber, listenerCfg *fgw.Listener) { switch listenerCfg.Protocol { case gwv1.HTTPProtocolType, gwv1.HTTPSProtocolType, gwv1.TLSProtocolType, gwv1.TCPProtocolType, gwv1.UDPProtocolType: - if len(e.Data) == 0 { + if len(e.data) == 0 { return } - for _, accessControl := range e.Data { - ac := accessControl - if !gwutils.IsRefToTarget(e.ReferenceGrants, &ac, ac.Spec.TargetRef, gw) { - continue - } + for _, accessControl := range e.data { + accessControl := accessControl.(*gwpav1alpha1.AccessControlPolicy) - if len(ac.Spec.Ports) == 0 { + if len(accessControl.Spec.Ports) == 0 { continue } - if c := accesscontrol.GetAccessControlConfigIfPortMatchesPolicy(port, ac); c != nil && listenerCfg.AccessControlLists == nil { + if c := accesscontrol.GetAccessControlConfigIfPortMatchesPolicy(port, accessControl); c != nil && listenerCfg.AccessControlLists == nil { listenerCfg.AccessControlLists = newAccessControlLists(c) break } } default: - log.Warn().Msgf("AccessControlPortEnricher: unsupported protocol %s", listenerCfg.Protocol) + log.Warn().Msgf("accessControlPortEnricher: unsupported protocol %s", listenerCfg.Protocol) } } diff --git a/pkg/gateway/policy/service_enrichers.go b/pkg/gateway/policy/service_enrichers.go index 51c17968b..d35ef3b78 100644 --- a/pkg/gateway/policy/service_enrichers.go +++ b/pkg/gateway/policy/service_enrichers.go @@ -1,10 +1,27 @@ package policy import ( + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/fields" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" + gwv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + gwpav1alpha1 "github.com/flomesh-io/fsm/pkg/apis/policyattachment/v1alpha1" "github.com/flomesh-io/fsm/pkg/gateway/fgw" + "github.com/flomesh-io/fsm/pkg/gateway/policy/utils/circuitbreaking" + "github.com/flomesh-io/fsm/pkg/gateway/policy/utils/healthcheck" + "github.com/flomesh-io/fsm/pkg/gateway/policy/utils/loadbalancer" + "github.com/flomesh-io/fsm/pkg/gateway/policy/utils/retry" + "github.com/flomesh-io/fsm/pkg/gateway/policy/utils/sessionsticky" + "github.com/flomesh-io/fsm/pkg/gateway/policy/utils/upstreamtls" + gwutils "github.com/flomesh-io/fsm/pkg/gateway/utils" ) +type ResolveServicePortNameFunc func(referer client.Object, ref gwv1alpha2.NamespacedPolicyTargetReference, port int32) *fgw.ServicePortName +type ResolveSecretFunc func(referer client.Object, ref gwv1.SecretObjectReference) (*corev1.Secret, error) + // ServicePolicyEnricher is an interface for enriching service level policies type ServicePolicyEnricher interface { // Enrich enriches the service config with the service level policy based on the service port name @@ -13,17 +30,46 @@ type ServicePolicyEnricher interface { // --- -// SessionStickyPolicyEnricher is an enricher for session sticky policies -type SessionStickyPolicyEnricher struct { - Data map[string]*gwpav1alpha1.SessionStickyConfig +func NewSessionStickyPolicyEnricher(cache cache.Cache, selector fields.Selector, targetRefToServicePortName ResolveServicePortNameFunc) ServicePolicyEnricher { + configs := make(map[string]*gwpav1alpha1.SessionStickyConfig) + + for _, sessionSticky := range gwutils.SortResources(gwutils.GetSessionStickies(cache, selector)) { + sessionSticky := sessionSticky.(*gwpav1alpha1.SessionStickyPolicy) + + for _, p := range sessionSticky.Spec.Ports { + if svcPortName := targetRefToServicePortName(sessionSticky, sessionSticky.Spec.TargetRef, int32(p.Port)); svcPortName != nil { + cfg := sessionsticky.ComputeSessionStickyConfig(p.Config, sessionSticky.Spec.DefaultConfig) + + if cfg == nil { + continue + } + + if _, ok := configs[svcPortName.String()]; ok { + log.Warn().Msgf("Policy is already defined for service port %s, SessionStickyPolicy %s/%s:%d will be dropped", svcPortName.String(), sessionSticky.Namespace, sessionSticky.Name, p.Port) + continue + } + + configs[svcPortName.String()] = cfg + } + } + } + + return &sessionStickyPolicyEnricher{ + data: configs, + } +} + +// sessionStickyPolicyEnricher is an enricher for session sticky policies +type sessionStickyPolicyEnricher struct { + data map[string]*gwpav1alpha1.SessionStickyConfig } -func (e *SessionStickyPolicyEnricher) Enrich(svcPortName string, svcCfg *fgw.ServiceConfig) { - if len(e.Data) == 0 { +func (e *sessionStickyPolicyEnricher) Enrich(svcPortName string, svcCfg *fgw.ServiceConfig) { + if len(e.data) == 0 { return } - if ssCfg, exists := e.Data[svcPortName]; exists { + if ssCfg, exists := e.data[svcPortName]; exists { svcCfg.StickyCookieName = ssCfg.CookieName svcCfg.StickyCookieExpires = ssCfg.Expires } @@ -31,68 +77,193 @@ func (e *SessionStickyPolicyEnricher) Enrich(svcPortName string, svcCfg *fgw.Ser // --- +func NewLoadBalancerPolicyEnricher(cache cache.Cache, selector fields.Selector, targetRefToServicePortName ResolveServicePortNameFunc) ServicePolicyEnricher { + loadBalancers := make(map[string]*gwpav1alpha1.LoadBalancerType) + + for _, lb := range gwutils.SortResources(gwutils.GetLoadBalancers(cache, selector)) { + lb := lb.(*gwpav1alpha1.LoadBalancerPolicy) + + for _, p := range lb.Spec.Ports { + if svcPortName := targetRefToServicePortName(lb, lb.Spec.TargetRef, int32(p.Port)); svcPortName != nil { + t := loadbalancer.ComputeLoadBalancerType(p.Type, lb.Spec.DefaultType) + + if t == nil { + continue + } + + if _, ok := loadBalancers[svcPortName.String()]; ok { + log.Warn().Msgf("Policy is already defined for service port %s, LoadBalancerPolicy %s/%s:%d will be dropped", svcPortName.String(), lb.Namespace, lb.Name, p.Port) + continue + } + + loadBalancers[svcPortName.String()] = t + } + } + } + + return &LoadBalancerPolicyEnricher{ + data: loadBalancers, + } +} + // LoadBalancerPolicyEnricher is an enricher for load balancer policies type LoadBalancerPolicyEnricher struct { - Data map[string]*gwpav1alpha1.LoadBalancerType + data map[string]*gwpav1alpha1.LoadBalancerType } func (e *LoadBalancerPolicyEnricher) Enrich(svcPortName string, svcCfg *fgw.ServiceConfig) { - if len(e.Data) == 0 { + if len(e.data) == 0 { return } - if lbType, exists := e.Data[svcPortName]; exists { + if lbType, exists := e.data[svcPortName]; exists { svcCfg.LoadBalancer = lbType } } // --- +func NewCircuitBreakingPolicyEnricher(cache cache.Cache, selector fields.Selector, targetRefToServicePortName ResolveServicePortNameFunc) ServicePolicyEnricher { + configs := make(map[string]*gwpav1alpha1.CircuitBreakingConfig) + + for _, cb := range gwutils.SortResources(gwutils.GetCircuitBreakings(cache, selector)) { + cb := cb.(*gwpav1alpha1.CircuitBreakingPolicy) + + for _, p := range cb.Spec.Ports { + if svcPortName := targetRefToServicePortName(cb, cb.Spec.TargetRef, int32(p.Port)); svcPortName != nil { + cfg := circuitbreaking.ComputeCircuitBreakingConfig(p.Config, cb.Spec.DefaultConfig) + + if cfg == nil { + continue + } + + if _, ok := configs[svcPortName.String()]; ok { + log.Warn().Msgf("Policy is already defined for service port %s, CircuitBreakingPolicy %s/%s:%d will be dropped", svcPortName.String(), cb.Namespace, cb.Name, p.Port) + continue + } + + configs[svcPortName.String()] = cfg + } + } + } + + return &CircuitBreakingPolicyEnricher{ + data: configs, + } +} + // CircuitBreakingPolicyEnricher is an enricher for circuit breaking policies type CircuitBreakingPolicyEnricher struct { - Data map[string]*gwpav1alpha1.CircuitBreakingConfig + data map[string]*gwpav1alpha1.CircuitBreakingConfig } func (e *CircuitBreakingPolicyEnricher) Enrich(svcPortName string, svcCfg *fgw.ServiceConfig) { - if len(e.Data) == 0 { + if len(e.data) == 0 { return } - if cbCfg, exists := e.Data[svcPortName]; exists { + if cbCfg, exists := e.data[svcPortName]; exists { svcCfg.CircuitBreaking = newCircuitBreaking(cbCfg) } } // --- +func NewHealthCheckPolicyEnricher(cache cache.Cache, selector fields.Selector, targetRefToServicePortName ResolveServicePortNameFunc) ServicePolicyEnricher { + configs := make(map[string]*gwpav1alpha1.HealthCheckConfig) + + for _, hc := range gwutils.SortResources(gwutils.GetHealthChecks(cache, selector)) { + hc := hc.(*gwpav1alpha1.HealthCheckPolicy) + + for _, p := range hc.Spec.Ports { + if svcPortName := targetRefToServicePortName(hc, hc.Spec.TargetRef, int32(p.Port)); svcPortName != nil { + cfg := healthcheck.ComputeHealthCheckConfig(p.Config, hc.Spec.DefaultConfig) + + if cfg == nil { + continue + } + + if _, ok := configs[svcPortName.String()]; ok { + log.Warn().Msgf("Policy is already defined for service port %s, HealthCheckPolicy %s/%s:%d will be dropped", svcPortName.String(), hc.Namespace, hc.Name, p.Port) + continue + } + + configs[svcPortName.String()] = cfg + } + } + } + + return &HealthCheckPolicyEnricher{ + data: configs, + } +} + // HealthCheckPolicyEnricher is an enricher for health check policies type HealthCheckPolicyEnricher struct { - Data map[string]*gwpav1alpha1.HealthCheckConfig + data map[string]*gwpav1alpha1.HealthCheckConfig } func (e *HealthCheckPolicyEnricher) Enrich(svcPortName string, svcCfg *fgw.ServiceConfig) { - if len(e.Data) == 0 { + if len(e.data) == 0 { return } - if hcCfg, exists := e.Data[svcPortName]; exists { + if hcCfg, exists := e.data[svcPortName]; exists { svcCfg.HealthCheck = newHealthCheck(hcCfg) } } // --- +func NewUpstreamTLSPolicyEnricher(cache cache.Cache, selector fields.Selector, targetRefToServicePortName ResolveServicePortNameFunc, secretRefToSecret ResolveSecretFunc) ServicePolicyEnricher { + configs := make(map[string]*UpstreamTLSConfig) + + for _, upstreamTLS := range gwutils.SortResources(gwutils.GetUpStreamTLSes(cache, selector)) { + upstreamTLS := upstreamTLS.(*gwpav1alpha1.UpstreamTLSPolicy) + + for _, p := range upstreamTLS.Spec.Ports { + if svcPortName := targetRefToServicePortName(upstreamTLS, upstreamTLS.Spec.TargetRef, int32(p.Port)); svcPortName != nil { + cfg := upstreamtls.ComputeUpstreamTLSConfig(p.Config, upstreamTLS.Spec.DefaultConfig) + + if cfg == nil { + continue + } + + secret, err := secretRefToSecret(upstreamTLS, cfg.CertificateRef) + if err != nil { + log.Error().Msgf("Failed to resolve Secret: %s", err) + continue + } + + if _, ok := configs[svcPortName.String()]; ok { + log.Warn().Msgf("Policy is already defined for service port %s, UpstreamTLSPolicy %s/%s:%d will be dropped", svcPortName.String(), upstreamTLS.Namespace, upstreamTLS.Name, p.Port) + continue + } + + configs[svcPortName.String()] = &UpstreamTLSConfig{ + MTLS: cfg.MTLS, + Secret: secret, + } + } + } + } + + return &UpstreamTLSPolicyEnricher{ + data: configs, + } +} + // UpstreamTLSPolicyEnricher is an enricher for upstream TLS policies type UpstreamTLSPolicyEnricher struct { - Data map[string]*UpstreamTLSConfig + data map[string]*UpstreamTLSConfig } func (e *UpstreamTLSPolicyEnricher) Enrich(svcPortName string, svcCfg *fgw.ServiceConfig) { - if len(e.Data) == 0 { + if len(e.data) == 0 { return } - if tlsCfg, exists := e.Data[svcPortName]; exists { + if tlsCfg, exists := e.data[svcPortName]; exists { svcCfg.UpstreamCert = newUpstreamCert(tlsCfg) svcCfg.MTLS = tlsCfg.MTLS } @@ -100,17 +271,46 @@ func (e *UpstreamTLSPolicyEnricher) Enrich(svcPortName string, svcCfg *fgw.Servi // --- +func NewRetryPolicyEnricher(cache cache.Cache, selector fields.Selector, targetRefToServicePortName ResolveServicePortNameFunc) ServicePolicyEnricher { + configs := make(map[string]*gwpav1alpha1.RetryConfig) + + for _, retryPolicy := range gwutils.SortResources(gwutils.GetRetries(cache, selector)) { + retryPolicy := retryPolicy.(*gwpav1alpha1.RetryPolicy) + + for _, p := range retryPolicy.Spec.Ports { + if svcPortName := targetRefToServicePortName(retryPolicy, retryPolicy.Spec.TargetRef, int32(p.Port)); svcPortName != nil { + cfg := retry.ComputeRetryConfig(p.Config, retryPolicy.Spec.DefaultConfig) + + if cfg == nil { + continue + } + + if _, ok := configs[svcPortName.String()]; ok { + log.Warn().Msgf("Policy is already defined for service port %s, RetryPolicy %s/%s:%d will be dropped", svcPortName.String(), retryPolicy.Namespace, retryPolicy.Name, p.Port) + continue + } + + configs[svcPortName.String()] = cfg + } + } + } + + return &RetryPolicyEnricher{ + data: configs, + } +} + // RetryPolicyEnricher is an enricher for retry policies type RetryPolicyEnricher struct { - Data map[string]*gwpav1alpha1.RetryConfig + data map[string]*gwpav1alpha1.RetryConfig } func (e *RetryPolicyEnricher) Enrich(svcPortName string, svcCfg *fgw.ServiceConfig) { - if len(e.Data) == 0 { + if len(e.data) == 0 { return } - if hcCfg, exists := e.Data[svcPortName]; exists { + if hcCfg, exists := e.data[svcPortName]; exists { svcCfg.Retry = newRetry(hcCfg) } } diff --git a/pkg/gateway/policy/status/utils.go b/pkg/gateway/policy/status/utils.go deleted file mode 100644 index a4842afe5..000000000 --- a/pkg/gateway/policy/status/utils.go +++ /dev/null @@ -1,74 +0,0 @@ -package status - -import ( - "time" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" - gwv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" -) - -// NotFoundCondition returns the not found condition with the given message for the policy -func NotFoundCondition(policy client.Object, message string) metav1.Condition { - return metav1.Condition{ - Type: string(gwv1alpha2.PolicyConditionAccepted), - Status: metav1.ConditionFalse, - ObservedGeneration: policy.GetGeneration(), - LastTransitionTime: metav1.Time{Time: time.Now()}, - Reason: string(gwv1alpha2.PolicyReasonTargetNotFound), - Message: message, - } -} - -// InvalidCondition returns the invalid condition with the given message for the policy -func InvalidCondition(policy client.Object, message string) metav1.Condition { - return metav1.Condition{ - Type: string(gwv1alpha2.PolicyConditionAccepted), - Status: metav1.ConditionFalse, - ObservedGeneration: policy.GetGeneration(), - LastTransitionTime: metav1.Time{Time: time.Now()}, - Reason: string(gwv1alpha2.PolicyReasonInvalid), - Message: message, - } -} - -// ConflictCondition returns the conflict condition with the given message for the policy -func ConflictCondition(policy client.Object, message string) metav1.Condition { - return metav1.Condition{ - Type: string(gwv1alpha2.PolicyConditionAccepted), - Status: metav1.ConditionFalse, - ObservedGeneration: policy.GetGeneration(), - LastTransitionTime: metav1.Time{Time: time.Now()}, - Reason: string(gwv1alpha2.PolicyReasonConflicted), - Message: message, - } -} - -// AcceptedCondition returns the accepted condition with the given message for the policy -func AcceptedCondition(policy client.Object) metav1.Condition { - return metav1.Condition{ - Type: string(gwv1alpha2.PolicyConditionAccepted), - Status: metav1.ConditionTrue, - ObservedGeneration: policy.GetGeneration(), - LastTransitionTime: metav1.Time{Time: time.Now()}, - Reason: string(gwv1alpha2.PolicyReasonAccepted), - Message: string(gwv1alpha2.PolicyReasonAccepted), - } -} - -// NoAccessCondition returns the no access condition with the given message for the policy -func NoAccessCondition(policy client.Object, message string) metav1.Condition { - return metav1.Condition{ - Type: string(gwv1alpha2.PolicyConditionAccepted), - Status: metav1.ConditionFalse, - ObservedGeneration: policy.GetGeneration(), - LastTransitionTime: metav1.Time{Time: time.Now()}, - Reason: "NoAccessToTarget", - Message: message, - } -} - -// ConditionPointer returns the pointer of the given condition -func ConditionPointer(condition metav1.Condition) *metav1.Condition { - return &condition -} diff --git a/pkg/gateway/policy/utils/accesscontrol/utils.go b/pkg/gateway/policy/utils/accesscontrol/utils.go index 4b134ecbb..401796b69 100644 --- a/pkg/gateway/policy/utils/accesscontrol/utils.go +++ b/pkg/gateway/policy/utils/accesscontrol/utils.go @@ -14,7 +14,7 @@ import ( ) // GetAccessControlConfigIfPortMatchesPolicy returns true if the port matches the access control policy -func GetAccessControlConfigIfPortMatchesPolicy(port gwv1.PortNumber, accessControlPolicy gwpav1alpha1.AccessControlPolicy) *gwpav1alpha1.AccessControlConfig { +func GetAccessControlConfigIfPortMatchesPolicy(port gwv1.PortNumber, accessControlPolicy *gwpav1alpha1.AccessControlPolicy) *gwpav1alpha1.AccessControlConfig { if len(accessControlPolicy.Spec.Ports) == 0 { return nil } @@ -29,7 +29,7 @@ func GetAccessControlConfigIfPortMatchesPolicy(port gwv1.PortNumber, accessContr } // GetAccessControlConfigIfRouteHostnameMatchesPolicy returns the access control config if the route hostname matches the policy -func GetAccessControlConfigIfRouteHostnameMatchesPolicy(routeHostname string, accessControlPolicy gwpav1alpha1.AccessControlPolicy) *gwpav1alpha1.AccessControlConfig { +func GetAccessControlConfigIfRouteHostnameMatchesPolicy(routeHostname string, accessControlPolicy *gwpav1alpha1.AccessControlPolicy) *gwpav1alpha1.AccessControlConfig { if len(accessControlPolicy.Spec.Hostnames) == 0 { return nil } @@ -57,7 +57,7 @@ func GetAccessControlConfigIfRouteHostnameMatchesPolicy(routeHostname string, ac } // GetAccessControlConfigIfHTTPRouteMatchesPolicy returns the access control config if the HTTP route matches the policy -func GetAccessControlConfigIfHTTPRouteMatchesPolicy(routeMatch gwv1.HTTPRouteMatch, accessControlPolicy gwpav1alpha1.AccessControlPolicy) *gwpav1alpha1.AccessControlConfig { +func GetAccessControlConfigIfHTTPRouteMatchesPolicy(routeMatch gwv1.HTTPRouteMatch, accessControlPolicy *gwpav1alpha1.AccessControlPolicy) *gwpav1alpha1.AccessControlConfig { if len(accessControlPolicy.Spec.HTTPAccessControls) == 0 { return nil } @@ -72,7 +72,7 @@ func GetAccessControlConfigIfHTTPRouteMatchesPolicy(routeMatch gwv1.HTTPRouteMat } // GetAccessControlConfigIfGRPCRouteMatchesPolicy returns the access control config if the GRPC route matches the policy -func GetAccessControlConfigIfGRPCRouteMatchesPolicy(routeMatch gwv1.GRPCRouteMatch, accessControlPolicy gwpav1alpha1.AccessControlPolicy) *gwpav1alpha1.AccessControlConfig { +func GetAccessControlConfigIfGRPCRouteMatchesPolicy(routeMatch gwv1.GRPCRouteMatch, accessControlPolicy *gwpav1alpha1.AccessControlPolicy) *gwpav1alpha1.AccessControlConfig { if len(accessControlPolicy.Spec.GRPCAccessControls) == 0 { return nil } diff --git a/pkg/gateway/policy/utils/faultinjection/utils.go b/pkg/gateway/policy/utils/faultinjection/utils.go index 77ae98e91..05a92e77a 100644 --- a/pkg/gateway/policy/utils/faultinjection/utils.go +++ b/pkg/gateway/policy/utils/faultinjection/utils.go @@ -14,7 +14,7 @@ import ( ) // GetFaultInjectionConfigIfRouteHostnameMatchesPolicy returns the fault injection config if the route hostname matches the policy -func GetFaultInjectionConfigIfRouteHostnameMatchesPolicy(routeHostname string, faultInjectionPolicy gwpav1alpha1.FaultInjectionPolicy) *gwpav1alpha1.FaultInjectionConfig { +func GetFaultInjectionConfigIfRouteHostnameMatchesPolicy(routeHostname string, faultInjectionPolicy *gwpav1alpha1.FaultInjectionPolicy) *gwpav1alpha1.FaultInjectionConfig { if len(faultInjectionPolicy.Spec.Hostnames) == 0 { return nil } @@ -42,7 +42,7 @@ func GetFaultInjectionConfigIfRouteHostnameMatchesPolicy(routeHostname string, f } // GetFaultInjectionConfigIfHTTPRouteMatchesPolicy returns the fault injection config if the HTTP route matches the policy -func GetFaultInjectionConfigIfHTTPRouteMatchesPolicy(routeMatch gwv1.HTTPRouteMatch, faultInjectionPolicy gwpav1alpha1.FaultInjectionPolicy) *gwpav1alpha1.FaultInjectionConfig { +func GetFaultInjectionConfigIfHTTPRouteMatchesPolicy(routeMatch gwv1.HTTPRouteMatch, faultInjectionPolicy *gwpav1alpha1.FaultInjectionPolicy) *gwpav1alpha1.FaultInjectionConfig { if len(faultInjectionPolicy.Spec.HTTPFaultInjections) == 0 { return nil } @@ -57,7 +57,7 @@ func GetFaultInjectionConfigIfHTTPRouteMatchesPolicy(routeMatch gwv1.HTTPRouteMa } // GetFaultInjectionConfigIfGRPCRouteMatchesPolicy returns the fault injection config if the GRPC route matches the policy -func GetFaultInjectionConfigIfGRPCRouteMatchesPolicy(routeMatch gwv1.GRPCRouteMatch, faultInjectionPolicy gwpav1alpha1.FaultInjectionPolicy) *gwpav1alpha1.FaultInjectionConfig { +func GetFaultInjectionConfigIfGRPCRouteMatchesPolicy(routeMatch gwv1.GRPCRouteMatch, faultInjectionPolicy *gwpav1alpha1.FaultInjectionPolicy) *gwpav1alpha1.FaultInjectionConfig { if len(faultInjectionPolicy.Spec.GRPCFaultInjections) == 0 { return nil } diff --git a/pkg/gateway/policy/utils/ratelimit/utils.go b/pkg/gateway/policy/utils/ratelimit/utils.go index 8b18ec3c2..6e53ff25b 100644 --- a/pkg/gateway/policy/utils/ratelimit/utils.go +++ b/pkg/gateway/policy/utils/ratelimit/utils.go @@ -14,7 +14,7 @@ import ( ) // GetRateLimitIfRouteHostnameMatchesPolicy returns the rate limit config if the route hostname matches the policy -func GetRateLimitIfRouteHostnameMatchesPolicy(routeHostname string, rateLimitPolicy gwpav1alpha1.RateLimitPolicy) *gwpav1alpha1.L7RateLimit { +func GetRateLimitIfRouteHostnameMatchesPolicy(routeHostname string, rateLimitPolicy *gwpav1alpha1.RateLimitPolicy) *gwpav1alpha1.L7RateLimit { if len(rateLimitPolicy.Spec.Hostnames) == 0 { return nil } @@ -42,7 +42,7 @@ func GetRateLimitIfRouteHostnameMatchesPolicy(routeHostname string, rateLimitPol } // GetRateLimitIfHTTPRouteMatchesPolicy returns the rate limit config if the HTTP route matches the policy -func GetRateLimitIfHTTPRouteMatchesPolicy(routeMatch gwv1.HTTPRouteMatch, rateLimitPolicy gwpav1alpha1.RateLimitPolicy) *gwpav1alpha1.L7RateLimit { +func GetRateLimitIfHTTPRouteMatchesPolicy(routeMatch gwv1.HTTPRouteMatch, rateLimitPolicy *gwpav1alpha1.RateLimitPolicy) *gwpav1alpha1.L7RateLimit { if len(rateLimitPolicy.Spec.HTTPRateLimits) == 0 { return nil } @@ -57,7 +57,7 @@ func GetRateLimitIfHTTPRouteMatchesPolicy(routeMatch gwv1.HTTPRouteMatch, rateLi } // GetRateLimitIfGRPCRouteMatchesPolicy returns the rate limit config if the GRPC route matches the policy -func GetRateLimitIfGRPCRouteMatchesPolicy(routeMatch gwv1.GRPCRouteMatch, rateLimitPolicy gwpav1alpha1.RateLimitPolicy) *gwpav1alpha1.L7RateLimit { +func GetRateLimitIfGRPCRouteMatchesPolicy(routeMatch gwv1.GRPCRouteMatch, rateLimitPolicy *gwpav1alpha1.RateLimitPolicy) *gwpav1alpha1.L7RateLimit { if len(rateLimitPolicy.Spec.GRPCRateLimits) == 0 { return nil } @@ -72,7 +72,7 @@ func GetRateLimitIfGRPCRouteMatchesPolicy(routeMatch gwv1.GRPCRouteMatch, rateLi } // GetRateLimitIfPortMatchesPolicy returns true if the port matches the rate limit policy -func GetRateLimitIfPortMatchesPolicy(port gwv1.PortNumber, rateLimitPolicy gwpav1alpha1.RateLimitPolicy) *int64 { +func GetRateLimitIfPortMatchesPolicy(port gwv1.PortNumber, rateLimitPolicy *gwpav1alpha1.RateLimitPolicy) *int64 { if len(rateLimitPolicy.Spec.Ports) == 0 { return nil } diff --git a/pkg/gateway/status/gw/gateway.go b/pkg/gateway/status/gw/gateway.go new file mode 100644 index 000000000..25538e572 --- /dev/null +++ b/pkg/gateway/status/gw/gateway.go @@ -0,0 +1,229 @@ +package gw + +import ( + "fmt" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" +) + +type GatewayStatusUpdate struct { + conditions map[gwv1.GatewayConditionType]metav1.Condition + existingConditions map[gwv1.GatewayConditionType]metav1.Condition + listenerStatus map[string]*gwv1.ListenerStatus + addresses []gwv1.GatewayStatusAddress + objectMeta *metav1.ObjectMeta + typeMeta *metav1.TypeMeta + resource client.Object + transitionTime metav1.Time + fullName types.NamespacedName + generation int64 +} + +func NewGatewayStatusUpdate(resource client.Object, meta *metav1.ObjectMeta, typeMeta *metav1.TypeMeta, gs *gwv1.GatewayStatus) *GatewayStatusUpdate { + return &GatewayStatusUpdate{ + objectMeta: meta, + typeMeta: typeMeta, + resource: resource, + transitionTime: metav1.Time{Time: time.Now()}, + fullName: types.NamespacedName{Namespace: meta.Namespace, Name: meta.Name}, + generation: meta.Generation, + existingConditions: getGatewayConditions(gs), + } +} + +// AddCondition returns a metav1.Condition for a given GatewayConditionType. +func (g *GatewayStatusUpdate) AddCondition( + conditionType gwv1.GatewayConditionType, + status metav1.ConditionStatus, + reason gwv1.GatewayConditionReason, + message string, +) metav1.Condition { + if g.conditions == nil { + g.conditions = make(map[gwv1.GatewayConditionType]metav1.Condition) + } + + if c, ok := g.conditions[conditionType]; ok { + message = fmt.Sprintf("%s, %s", c.Message, message) + } + + newCond := metav1.Condition{ + Reason: string(reason), + Status: status, + Type: string(conditionType), + Message: message, + LastTransitionTime: metav1.NewTime(time.Now()), + ObservedGeneration: g.generation, + } + g.conditions[conditionType] = newCond + + return newCond +} + +func (g *GatewayStatusUpdate) ConditionExists(conditionType gwv1.GatewayConditionType) bool { + _, ok := g.conditions[conditionType] + return ok +} + +func (g *GatewayStatusUpdate) SetAddresses(addresses []gwv1.GatewayStatusAddress) { + g.addresses = addresses +} + +func (g *GatewayStatusUpdate) SetListenerSupportedKinds(listenerName string, groupKinds []gwv1.RouteGroupKind) { + if g.listenerStatus == nil { + g.listenerStatus = map[string]*gwv1.ListenerStatus{} + } + if g.listenerStatus[listenerName] == nil { + g.listenerStatus[listenerName] = &gwv1.ListenerStatus{ + Name: gwv1.SectionName(listenerName), + } + } + + g.listenerStatus[listenerName].SupportedKinds = append(g.listenerStatus[listenerName].SupportedKinds, groupKinds...) +} + +func (g *GatewayStatusUpdate) SetListenerAttachedRoutes(listenerName string, numRoutes int) { + if g.listenerStatus == nil { + g.listenerStatus = map[string]*gwv1.ListenerStatus{} + } + if g.listenerStatus[listenerName] == nil { + g.listenerStatus[listenerName] = &gwv1.ListenerStatus{ + Name: gwv1.SectionName(listenerName), + } + } + + g.listenerStatus[listenerName].AttachedRoutes = int32(numRoutes) +} + +// AddListenerCondition adds a Condition for the specified listener. +func (g *GatewayStatusUpdate) AddListenerCondition( + listenerName string, + cond gwv1.ListenerConditionType, + status metav1.ConditionStatus, + reason gwv1.ListenerConditionReason, + message string, +) metav1.Condition { + if g.listenerStatus == nil { + g.listenerStatus = map[string]*gwv1.ListenerStatus{} + } + if g.listenerStatus[listenerName] == nil { + g.listenerStatus[listenerName] = &gwv1.ListenerStatus{ + Name: gwv1.SectionName(listenerName), + } + } + + listenerStatus := g.listenerStatus[listenerName] + + idx := -1 + for i, existing := range listenerStatus.Conditions { + if existing.Type == string(cond) { + idx = i + message = fmt.Sprintf("%s, %s", existing.Message, message) + break + } + } + + newCond := metav1.Condition{ + Reason: string(reason), + Status: status, + Type: string(cond), + Message: message, + LastTransitionTime: metav1.NewTime(time.Now()), + ObservedGeneration: g.generation, + } + + if idx > -1 { + listenerStatus.Conditions[idx] = newCond + } else { + listenerStatus.Conditions = append(listenerStatus.Conditions, newCond) + } + + return newCond +} + +func getGatewayConditions(gs *gwv1.GatewayStatus) map[gwv1.GatewayConditionType]metav1.Condition { + conditions := make(map[gwv1.GatewayConditionType]metav1.Condition) + for _, cond := range gs.Conditions { + if _, ok := conditions[gwv1.GatewayConditionType(cond.Type)]; !ok { + conditions[gwv1.GatewayConditionType(cond.Type)] = cond + } + } + return conditions +} + +func (g *GatewayStatusUpdate) GetListenerStatus(listenerName string) *gwv1.ListenerStatus { + if g.listenerStatus == nil { + return nil + } + + return g.listenerStatus[listenerName] +} + +func (g *GatewayStatusUpdate) Mutate(obj client.Object) client.Object { + o, ok := obj.(*gwv1.Gateway) + if !ok { + panic(fmt.Sprintf("Unsupported %T object %s/%s in GatewayStatusUpdate status mutator", + obj, g.fullName.Namespace, g.fullName.Name, + )) + } + + updated := o.DeepCopy() + + var conditionsToWrite []metav1.Condition + + for _, cond := range g.conditions { + // Set the Condition's observed generation based on + // the generation of the gateway we looked at. + cond.ObservedGeneration = g.generation + cond.LastTransitionTime = g.transitionTime + + // is there a newer Condition on the gateway matching + // this condition's type? If so, our observation is stale, + // so don't write it, keep the newer one instead. + var newerConditionExists bool + for _, existingCond := range g.existingConditions { + if existingCond.Type != cond.Type { + continue + } + + if existingCond.ObservedGeneration > cond.ObservedGeneration { + conditionsToWrite = append(conditionsToWrite, existingCond) + newerConditionExists = true + break + } + } + + // if we didn't find a newer version of the Condition on the + // gateway, then write the one we computed. + if !newerConditionExists { + conditionsToWrite = append(conditionsToWrite, cond) + } + } + + updated.Status.Conditions = conditionsToWrite + + // Overwrite all listener statuses since we re-compute all of them + // for each Gateway status update. + var listenerStatusToWrite []gwv1.ListenerStatus + for _, status := range g.listenerStatus { + if status.Conditions == nil { + // Conditions is a required field so we have to specify an empty slice here + status.Conditions = []metav1.Condition{} + } + if status.SupportedKinds == nil { + // SupportedKinds is a required field so we have to specify an empty slice here + status.SupportedKinds = []gwv1.RouteGroupKind{} + } + listenerStatusToWrite = append(listenerStatusToWrite, *status) + } + + updated.Status.Listeners = listenerStatusToWrite + + // Gateway addresses + updated.Status.Addresses = g.addresses + + return updated +} diff --git a/pkg/gateway/status/policy/policy.go b/pkg/gateway/status/policy/policy.go new file mode 100644 index 000000000..594f57ef7 --- /dev/null +++ b/pkg/gateway/status/policy/policy.go @@ -0,0 +1,122 @@ +package policy + +import ( + "fmt" + "time" + + metautil "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + gwv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + + gwpav1alpha1 "github.com/flomesh-io/fsm/pkg/apis/policyattachment/v1alpha1" +) + +// PolicyUpdate is a struct that holds the information needed to update the status of a policy attachment +type PolicyUpdate struct { + objectMeta *metav1.ObjectMeta + typeMeta *metav1.TypeMeta + targetRef gwv1alpha2.NamespacedPolicyTargetReference + resource client.Object + transitionTime metav1.Time + fullName types.NamespacedName + generation int64 + conditions []metav1.Condition +} + +func NewPolicyUpdate(resource client.Object, meta *metav1.ObjectMeta, typeMeta *metav1.TypeMeta, targetRef gwv1alpha2.NamespacedPolicyTargetReference, conditions []metav1.Condition) *PolicyUpdate { + return &PolicyUpdate{ + objectMeta: meta, + typeMeta: typeMeta, + resource: resource, + transitionTime: metav1.Now(), + fullName: types.NamespacedName{Namespace: meta.Namespace, Name: meta.Name}, + generation: meta.Generation, + targetRef: targetRef, + conditions: conditions, + } +} + +func (r *PolicyUpdate) AddCondition(condition metav1.Condition) metav1.Condition { + cond := metav1.Condition{ + Reason: condition.Reason, + Status: condition.Status, + Type: condition.Type, + Message: condition.Message, + LastTransitionTime: metav1.NewTime(time.Now()), + ObservedGeneration: r.generation, + } + + metautil.SetStatusCondition(&r.conditions, cond) + + return cond +} + +func (r *PolicyUpdate) ConditionExists(conditionType gwv1alpha2.PolicyConditionType) bool { + for _, c := range r.conditions { + if c.Type == string(conditionType) { + return true + } + } + return false +} + +func (r *PolicyUpdate) Conditions() []metav1.Condition { + return r.conditions +} + +func (r *PolicyUpdate) GetTargetRef() gwv1alpha2.NamespacedPolicyTargetReference { + return r.targetRef +} + +func (r *PolicyUpdate) GetResource() client.Object { + return r.resource +} + +func (r *PolicyUpdate) GetFullName() types.NamespacedName { + return r.fullName +} + +func (r *PolicyUpdate) Mutate(obj client.Object) client.Object { + switch o := obj.(type) { + case *gwpav1alpha1.AccessControlPolicy: + policy := o.DeepCopy() + policy.Status.Conditions = r.conditions + return policy + case *gwpav1alpha1.RateLimitPolicy: + policy := o.DeepCopy() + policy.Status.Conditions = r.conditions + return policy + case *gwpav1alpha1.FaultInjectionPolicy: + policy := o.DeepCopy() + policy.Status.Conditions = r.conditions + return policy + case *gwpav1alpha1.SessionStickyPolicy: + policy := o.DeepCopy() + policy.Status.Conditions = r.conditions + return policy + case *gwpav1alpha1.CircuitBreakingPolicy: + policy := o.DeepCopy() + policy.Status.Conditions = r.conditions + return policy + case *gwpav1alpha1.LoadBalancerPolicy: + policy := o.DeepCopy() + policy.Status.Conditions = r.conditions + return policy + case *gwpav1alpha1.HealthCheckPolicy: + policy := o.DeepCopy() + policy.Status.Conditions = r.conditions + return policy + case *gwpav1alpha1.RetryPolicy: + policy := o.DeepCopy() + policy.Status.Conditions = r.conditions + return policy + case *gwpav1alpha1.UpstreamTLSPolicy: + policy := o.DeepCopy() + policy.Status.Conditions = r.conditions + return policy + default: + panic(fmt.Sprintf("Unsupported %T object %s/%s in PolicyUpdate status mutator", obj, r.fullName.Namespace, r.fullName.Name)) + } +} diff --git a/pkg/gateway/policy/status/policy_status_processor.go b/pkg/gateway/status/policy/policy_status_processor.go similarity index 61% rename from pkg/gateway/policy/status/policy_status_processor.go rename to pkg/gateway/status/policy/policy_status_processor.go index 1f9bcbaf5..078f49f3e 100644 --- a/pkg/gateway/policy/status/policy_status_processor.go +++ b/pkg/gateway/status/policy/policy_status_processor.go @@ -1,4 +1,4 @@ -package status +package policy import ( "context" @@ -6,6 +6,11 @@ import ( "sort" "strings" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/flomesh-io/fsm/pkg/gateway/status" + "github.com/flomesh-io/fsm/pkg/gateway/status/route" + gwpkg "github.com/flomesh-io/fsm/pkg/gateway/types" "github.com/flomesh-io/fsm/pkg/k8s/informers" @@ -16,12 +21,9 @@ import ( "k8s.io/apimachinery/pkg/types" "github.com/flomesh-io/fsm/pkg/constants" - gwtypes "github.com/flomesh-io/fsm/pkg/gateway/types" gwutils "github.com/flomesh-io/fsm/pkg/gateway/utils" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" - gwv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" ) // PolicyStatusProcessor is a processor for processing port level policy status @@ -40,13 +42,13 @@ type PolicyStatusProcessor struct { type GetObjectByGroupKindFunc func(group gwv1.Group, kind gwv1.Kind) client.Object // GetPoliciesFunc returns the policies, and returns condition if there is any error -type GetPoliciesFunc func(policy client.Object, target client.Object) (map[gwpkg.PolicyMatchType][]client.Object, *metav1.Condition) +type GetPoliciesFunc func(target client.Object) map[gwpkg.PolicyMatchType][]client.Object // FindConflictPortFunc finds the conflicted port level policy type FindConflictPortFunc func(gateway *gwv1.Gateway, policy client.Object, allPortLevelPolicies []client.Object) *types.NamespacedName // FindConflictedHostnamesBasedPolicyFunc finds the conflicted hostnames based policy -type FindConflictedHostnamesBasedPolicyFunc func(route *gwtypes.RouteContext, policy client.Object, allHostnamesBasedPolicies []client.Object) *types.NamespacedName +type FindConflictedHostnamesBasedPolicyFunc func(route status.RouteStatusObject, parentRefs []gwv1.ParentReference, policy client.Object, allHostnamesBasedPolicies []client.Object) *types.NamespacedName // FindConflictedHTTPRouteBasedPolicyFunc finds the conflicted HTTPRoute based policy type FindConflictedHTTPRouteBasedPolicyFunc func(route *gwv1.HTTPRoute, policy client.Object, allRouteBasedPolicies []client.Object) *types.NamespacedName @@ -55,85 +57,99 @@ type FindConflictedHTTPRouteBasedPolicyFunc func(route *gwv1.HTTPRoute, policy c type FindConflictedGRPCRouteBasedPolicyFunc func(route *gwv1.GRPCRoute, policy client.Object, allRouteBasedPolicies []client.Object) *types.NamespacedName // Process processes the policy status of port, hostnames and route level -func (p *PolicyStatusProcessor) Process(ctx context.Context, policy client.Object, targetRef gwv1alpha2.NamespacedPolicyTargetReference) metav1.Condition { +func (p *PolicyStatusProcessor) Process(ctx context.Context, updater status.Updater, u *PolicyUpdate) { + p.internalProcess(ctx, u) + + updater.Send(status.Update{ + Resource: u.GetResource(), + NamespacedName: u.GetFullName(), + Mutator: u, + }) +} + +func (p *PolicyStatusProcessor) internalProcess(ctx context.Context, update *PolicyUpdate) metav1.Condition { + policy := update.GetResource() + targetRef := update.GetTargetRef() + _, ok := p.getGatewayAPIGroupKindObjectMapping()[string(targetRef.Group)] if !ok { - return InvalidCondition(policy, fmt.Sprintf("Invalid target reference group, only %q is/are supported", strings.Join(p.supportedGroups(), ","))) + return update.AddCondition(invalidCondition(fmt.Sprintf("Invalid target reference group, only %q is/are supported", strings.Join(p.supportedGroups(), ",")))) } target := p.getGatewayAPIObjectByGroupKind(targetRef.Group, targetRef.Kind) if target == nil { - return InvalidCondition(policy, fmt.Sprintf("Invalid target reference kind, only %q are supported", strings.Join(p.supportedKinds(), ","))) - } - - referenceGrants := p.Informer.GetGatewayResourcesFromCache(informers.ReferenceGrantResourceType, false) - if !gwutils.HasAccessToTargetRef(policy, targetRef, referenceGrants) { - return NoAccessCondition(policy, fmt.Sprintf("Cross namespace reference to target %s/%s/%s is not allowed", targetRef.Kind, ns(targetRef.Namespace), targetRef.Name)) + return update.AddCondition(invalidCondition(fmt.Sprintf("Invalid target reference kind, only %q are supported", strings.Join(p.supportedKinds(), ",")))) } key := types.NamespacedName{ - Namespace: gwutils.Namespace(targetRef.Namespace, policy.GetNamespace()), + Namespace: gwutils.NamespaceDerefOr(targetRef.Namespace, policy.GetNamespace()), Name: string(targetRef.Name), } if err := p.Get(ctx, key, target); err != nil { if errors.IsNotFound(err) { - return NotFoundCondition(policy, fmt.Sprintf("Invalid target reference, cannot find target %s %q", targetRef.Kind, key.String())) + return update.AddCondition(notFoundCondition(fmt.Sprintf("Invalid target reference, cannot find target %s %q", targetRef.Kind, key.String()))) } else { - return InvalidCondition(policy, fmt.Sprintf("Failed to get target %s %q: %s", targetRef.Kind, key, err)) + return update.AddCondition(invalidCondition(fmt.Sprintf("Failed to get target %s %q: %s", targetRef.Kind, key, err))) } } - policies, condition := p.getSortedPolices(policy, target) - if condition != nil { - return *condition - } + policies := p.getSortedPolices(target) switch obj := target.(type) { case *gwv1.Gateway: if p.FindConflictPort != nil && len(policies[gwpkg.PolicyMatchTypePort]) > 0 { if conflict := p.FindConflictPort(obj, policy, policies[gwpkg.PolicyMatchTypePort]); conflict != nil { - return ConflictCondition(policy, fmt.Sprintf("Conflict with %s: %s", policy.GetObjectKind().GroupVersionKind().Kind, conflict)) + return update.AddCondition(conflictCondition(fmt.Sprintf("Conflict with %s: %s", policy.GetObjectKind().GroupVersionKind().Kind, conflict))) } } case *gwv1.HTTPRoute: if p.FindConflictedHostnamesBasedPolicy != nil && len(policies[gwpkg.PolicyMatchTypeHostnames]) > 0 { - info := gwutils.ToRouteContext(obj) - - if conflict := p.FindConflictedHostnamesBasedPolicy(info, policy, policies[gwpkg.PolicyMatchTypeHostnames]); conflict != nil { - return ConflictCondition(policy, fmt.Sprintf("Conflict with %s: %s", policy.GetObjectKind().GroupVersionKind().Kind, conflict)) + info := route.NewRouteStatusHolder( + obj, + &obj.ObjectMeta, + &obj.TypeMeta, + obj.Spec.Hostnames, + gwutils.ToSlicePtr(obj.Status.Parents), + ) + + if conflict := p.FindConflictedHostnamesBasedPolicy(info, obj.Spec.ParentRefs, policy, policies[gwpkg.PolicyMatchTypeHostnames]); conflict != nil { + return update.AddCondition(conflictCondition(fmt.Sprintf("Conflict with %s: %s", policy.GetObjectKind().GroupVersionKind().Kind, conflict))) } } if p.FindConflictedHTTPRouteBasedPolicy != nil && len(policies[gwpkg.PolicyMatchTypeHTTPRoute]) > 0 { if conflict := p.FindConflictedHTTPRouteBasedPolicy(obj, policy, policies[gwpkg.PolicyMatchTypeHTTPRoute]); conflict != nil { - return ConflictCondition(policy, fmt.Sprintf("Conflict with %s: %s", policy.GetObjectKind().GroupVersionKind().Kind, conflict)) + return update.AddCondition(conflictCondition(fmt.Sprintf("Conflict with %s: %s", policy.GetObjectKind().GroupVersionKind().Kind, conflict))) } } case *gwv1.GRPCRoute: if p.FindConflictedHostnamesBasedPolicy != nil && len(policies[gwpkg.PolicyMatchTypeHostnames]) > 0 { - info := gwutils.ToRouteContext(obj) - - if conflict := p.FindConflictedHostnamesBasedPolicy(info, policy, policies[gwpkg.PolicyMatchTypeHostnames]); conflict != nil { - return ConflictCondition(policy, fmt.Sprintf("Conflict with %s: %s", policy.GetObjectKind().GroupVersionKind().Kind, conflict)) + info := route.NewRouteStatusHolder( + obj, + &obj.ObjectMeta, + &obj.TypeMeta, + obj.Spec.Hostnames, + gwutils.ToSlicePtr(obj.Status.Parents), + ) + + if conflict := p.FindConflictedHostnamesBasedPolicy(info, obj.Spec.ParentRefs, policy, policies[gwpkg.PolicyMatchTypeHostnames]); conflict != nil { + return update.AddCondition(conflictCondition(fmt.Sprintf("Conflict with %s: %s", policy.GetObjectKind().GroupVersionKind().Kind, conflict))) } } if p.FindConflictedGRPCRouteBasedPolicy != nil && len(policies[gwpkg.PolicyMatchTypeGRPCRoute]) > 0 { if conflict := p.FindConflictedGRPCRouteBasedPolicy(obj, policy, policies[gwpkg.PolicyMatchTypeGRPCRoute]); conflict != nil { - return ConflictCondition(policy, fmt.Sprintf("Conflict with %s: %s", policy.GetObjectKind().GroupVersionKind().Kind, conflict)) + return update.AddCondition(conflictCondition(fmt.Sprintf("Conflict with %s: %s", policy.GetObjectKind().GroupVersionKind().Kind, conflict))) } } } - return AcceptedCondition(policy) + return update.AddCondition(acceptedCondition()) } -func (p *PolicyStatusProcessor) getSortedPolices(policy client.Object, target client.Object) (map[gwpkg.PolicyMatchType][]client.Object, *metav1.Condition) { - policies, condition := p.GetPolicies(policy, target) - if condition != nil { - return nil, condition - } +func (p *PolicyStatusProcessor) getSortedPolices(target client.Object) map[gwpkg.PolicyMatchType][]client.Object { + policies := p.GetPolicies(target) // sort each type of by creation timestamp, then by namespace/name for matchType, ps := range policies { @@ -147,7 +163,7 @@ func (p *PolicyStatusProcessor) getSortedPolices(policy client.Object, target cl policies[matchType] = ps } - return policies, nil + return policies } func (p *PolicyStatusProcessor) getGatewayAPIObjectByGroupKind(group gwv1.Group, kind gwv1.Kind) client.Object { @@ -198,11 +214,3 @@ func (p *PolicyStatusProcessor) supportedKinds() []string { return kinds } - -func ns(namespace *gwv1.Namespace) string { - if namespace == nil { - return "" - } - - return string(*namespace) -} diff --git a/pkg/gateway/policy/status/service_policy_status_processor.go b/pkg/gateway/status/policy/service_policy_status_processor.go similarity index 70% rename from pkg/gateway/policy/status/service_policy_status_processor.go rename to pkg/gateway/status/policy/service_policy_status_processor.go index d2659dab4..9db05aebf 100644 --- a/pkg/gateway/policy/status/service_policy_status_processor.go +++ b/pkg/gateway/status/policy/service_policy_status_processor.go @@ -1,4 +1,4 @@ -package status +package policy import ( "context" @@ -6,6 +6,8 @@ import ( "sort" "strings" + "github.com/flomesh-io/fsm/pkg/gateway/status" + "github.com/flomesh-io/fsm/pkg/k8s/informers" gwv1 "sigs.k8s.io/gateway-api/apis/v1" @@ -19,8 +21,6 @@ import ( "github.com/flomesh-io/fsm/pkg/constants" gwutils "github.com/flomesh-io/fsm/pkg/gateway/utils" - gwv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -35,42 +35,50 @@ type ServicePolicyStatusProcessor struct { } // GetAttachedPoliciesFunc is a function for getting attached policies for a service -type GetAttachedPoliciesFunc func(policy client.Object, svc client.Object) ([]client.Object, *metav1.Condition) +type GetAttachedPoliciesFunc func(svc client.Object) ([]client.Object, *metav1.Condition) // FindConflictFunc is a function for finding conflicted policy for a service port type FindConflictFunc func(policy client.Object, allPolicies []client.Object, port int32) *types.NamespacedName // Process processes the service level policy status -func (p *ServicePolicyStatusProcessor) Process(ctx context.Context, policy client.Object, targetRef gwv1alpha2.NamespacedPolicyTargetReference) metav1.Condition { +func (p *ServicePolicyStatusProcessor) Process(ctx context.Context, updater status.Updater, u *PolicyUpdate) { + p.internalProcess(ctx, u) + + updater.Send(status.Update{ + Resource: u.GetResource(), + NamespacedName: u.GetFullName(), + Mutator: u, + }) +} + +func (p *ServicePolicyStatusProcessor) internalProcess(ctx context.Context, u *PolicyUpdate) metav1.Condition { + policy := u.GetResource() + targetRef := u.GetTargetRef() + _, ok := p.getServiceGroupKindObjectMapping()[string(targetRef.Group)] if !ok { - return InvalidCondition(policy, fmt.Sprintf("Invalid target reference group %q, only %q is/are supported", targetRef.Group, strings.Join(p.supportedGroups(), ","))) + return u.AddCondition(invalidCondition(fmt.Sprintf("Invalid target reference group %q, only %q is/are supported", targetRef.Group, strings.Join(p.supportedGroups(), ",")))) } svc := p.getServiceObjectByGroupKind(targetRef.Group, targetRef.Kind) if svc == nil { - return InvalidCondition(policy, fmt.Sprintf("Invalid target reference kind %q, only %q are supported", targetRef.Kind, strings.Join(p.supportedKinds(), ","))) - } - - referenceGrants := p.Informer.GetGatewayResourcesFromCache(informers.ReferenceGrantResourceType, false) - if !gwutils.HasAccessToTargetRef(policy, targetRef, referenceGrants) { - return NoAccessCondition(policy, fmt.Sprintf("Cross namespace reference to target %s/%s/%s is not allowed", targetRef.Kind, ns(targetRef.Namespace), targetRef.Name)) + return u.AddCondition(invalidCondition(fmt.Sprintf("Invalid target reference kind %q, only %q are supported", targetRef.Kind, strings.Join(p.supportedKinds(), ",")))) } key := types.NamespacedName{ - Namespace: gwutils.Namespace(targetRef.Namespace, policy.GetNamespace()), + Namespace: gwutils.NamespaceDerefOr(targetRef.Namespace, policy.GetNamespace()), Name: string(targetRef.Name), } if err := p.Get(ctx, key, svc); err != nil { if errors.IsNotFound(err) { - return NotFoundCondition(policy, fmt.Sprintf("Invalid target reference, cannot find target %s %q", targetRef.Kind, key.String())) + return u.AddCondition(notFoundCondition(fmt.Sprintf("Invalid target reference, cannot find target %s %q", targetRef.Kind, key.String()))) } else { - return InvalidCondition(policy, fmt.Sprintf("Failed to get target %s %q: %s", targetRef.Kind, key, err)) + return u.AddCondition(invalidCondition(fmt.Sprintf("Failed to get target %s %q: %s", targetRef.Kind, key, err))) } } - policies, condition := p.getSortedAttachedPolices(policy, svc) + policies, condition := p.getSortedAttachedPolices(svc) if condition != nil { return *condition } @@ -78,19 +86,19 @@ func (p *ServicePolicyStatusProcessor) Process(ctx context.Context, policy clien switch svc := svc.(type) { case *corev1.Service: if conflict := p.getConflictedPolicyByService(policy, policies, svc); conflict != nil { - return ConflictCondition(policy, fmt.Sprintf("Conflict with %s: %s", policy.GetObjectKind().GroupVersionKind().Kind, conflict)) + return u.AddCondition(conflictCondition(fmt.Sprintf("Conflict with %s: %s", policy.GetObjectKind().GroupVersionKind().Kind, conflict))) } case *mcsv1alpha1.ServiceImport: if conflict := p.getConflictedPolicyByServiceImport(policy, policies, svc); conflict != nil { - return ConflictCondition(policy, fmt.Sprintf("Conflict with %s: %s", policy.GetObjectKind().GroupVersionKind().Kind, conflict)) + return u.AddCondition(conflictCondition(fmt.Sprintf("Conflict with %s: %s", policy.GetObjectKind().GroupVersionKind().Kind, conflict))) } } - return AcceptedCondition(policy) + return u.AddCondition(acceptedCondition()) } -func (p *ServicePolicyStatusProcessor) getSortedAttachedPolices(policy client.Object, svc client.Object) ([]client.Object, *metav1.Condition) { - policies, condition := p.GetAttachedPolicies(policy, svc) +func (p *ServicePolicyStatusProcessor) getSortedAttachedPolices(svc client.Object) ([]client.Object, *metav1.Condition) { + policies, condition := p.GetAttachedPolicies(svc) if condition != nil { return nil, condition } diff --git a/pkg/gateway/policy/status/types.go b/pkg/gateway/status/policy/types.go similarity index 98% rename from pkg/gateway/policy/status/types.go rename to pkg/gateway/status/policy/types.go index eee08b24f..ce874f63b 100644 --- a/pkg/gateway/policy/status/types.go +++ b/pkg/gateway/status/policy/types.go @@ -1,4 +1,4 @@ -package status +package policy import ( corev1 "k8s.io/api/core/v1" diff --git a/pkg/gateway/status/policy/utils.go b/pkg/gateway/status/policy/utils.go new file mode 100644 index 000000000..e005ad4af --- /dev/null +++ b/pkg/gateway/status/policy/utils.go @@ -0,0 +1,56 @@ +package policy + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + gwv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" +) + +// notFoundCondition returns the not found condition with the given message for the policy +func notFoundCondition(message string) metav1.Condition { + return metav1.Condition{ + Type: string(gwv1alpha2.PolicyConditionAccepted), + Status: metav1.ConditionFalse, + Reason: string(gwv1alpha2.PolicyReasonTargetNotFound), + Message: message, + } +} + +// invalidCondition returns the invalid condition with the given message for the policy +func invalidCondition(message string) metav1.Condition { + return metav1.Condition{ + Type: string(gwv1alpha2.PolicyConditionAccepted), + Status: metav1.ConditionFalse, + Reason: string(gwv1alpha2.PolicyReasonInvalid), + Message: message, + } +} + +// conflictCondition returns the conflict condition with the given message for the policy +func conflictCondition(message string) metav1.Condition { + return metav1.Condition{ + Type: string(gwv1alpha2.PolicyConditionAccepted), + Status: metav1.ConditionFalse, + Reason: string(gwv1alpha2.PolicyReasonConflicted), + Message: message, + } +} + +// acceptedCondition returns the accepted condition with the given message for the policy +func acceptedCondition() metav1.Condition { + return metav1.Condition{ + Type: string(gwv1alpha2.PolicyConditionAccepted), + Status: metav1.ConditionTrue, + Reason: string(gwv1alpha2.PolicyReasonAccepted), + Message: string(gwv1alpha2.PolicyReasonAccepted), + } +} + +// noAccessCondition returns the no access condition with the given message for the policy +//func noAccessCondition(message string) metav1.Condition { +// return metav1.Condition{ +// Type: string(gwv1alpha2.PolicyConditionAccepted), +// Status: metav1.ConditionFalse, +// Reason: "NoAccessToTarget", +// Message: message, +// } +//} diff --git a/pkg/gateway/status/route.go b/pkg/gateway/status/route.go deleted file mode 100644 index 20e51bcb3..000000000 --- a/pkg/gateway/status/route.go +++ /dev/null @@ -1,148 +0,0 @@ -/* - * MIT License - * - * Copyright (c) since 2021, flomesh.io Authors. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package status - -import ( - "context" - "fmt" - "time" - - gwtypes "github.com/flomesh-io/fsm/pkg/gateway/types" - - metautil "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" - gwv1 "sigs.k8s.io/gateway-api/apis/v1" - - "github.com/flomesh-io/fsm/pkg/constants" - gwutils "github.com/flomesh-io/fsm/pkg/gateway/utils" - "github.com/flomesh-io/fsm/pkg/k8s/informers" -) - -// RouteStatusProcessor is responsible for computing the status of a Route -type RouteStatusProcessor struct { - Informers *informers.InformerCollection -} - -// ProcessRouteStatus computes the status of a Route -func (p *RouteStatusProcessor) ProcessRouteStatus(_ context.Context, route client.Object) ([]gwv1.RouteParentStatus, error) { - activeGateways := gwutils.GetActiveGateways(p.Informers.GetGatewayResourcesFromCache(informers.GatewaysResourceType, false)) - - if len(activeGateways) > 0 { - params := gwutils.ToRouteContext(route) - if params == nil { - return nil, fmt.Errorf("failed to convert route to route context, unsupported route type %T", route) - } - - return p.computeRouteParentStatus(activeGateways, params), nil - } - - return nil, nil -} - -func (p *RouteStatusProcessor) computeRouteParentStatus( - activeGateways []*gwv1.Gateway, - params *gwtypes.RouteContext, -) []gwv1.RouteParentStatus { - status := make([]gwv1.RouteParentStatus, 0) - - for _, gw := range activeGateways { - validListeners := gwutils.GetValidListenersForGateway(gw) - - for _, parentRef := range params.ParentRefs { - if !gwutils.IsRefToGateway(parentRef, gwutils.ObjectKey(gw)) { - continue - } - - routeParentStatus := gwv1.RouteParentStatus{ - ParentRef: parentRef, - ControllerName: constants.GatewayController, - Conditions: make([]metav1.Condition, 0), - } - - //allowedListeners := p.getAllowedListenersAndSetStatus(gw, parentRef, params, validListeners, routeParentStatus) - allowedListeners, conditions := gwutils.GetAllowedListeners(p.Informers.GetListers().Namespace, gw, parentRef, params, validListeners) - //if len(allowedListeners) == 0 { - // - //} - if len(conditions) > 0 { - for _, condition := range conditions { - metautil.SetStatusCondition(&routeParentStatus.Conditions, condition) - } - } - - count := 0 - for _, listener := range allowedListeners { - hostnames := gwutils.GetValidHostnames(listener.Hostname, params.Hostnames) - - //if len(hostnames) == 0 { - // continue - //} - - count += len(hostnames) - } - - switch params.GVK.Kind { - case constants.GatewayAPIHTTPRouteKind, constants.GatewayAPITLSRouteKind, constants.GatewayAPIGRPCRouteKind: - if count == 0 && metautil.FindStatusCondition(routeParentStatus.Conditions, string(gwv1.RouteConditionAccepted)) == nil { - metautil.SetStatusCondition(&routeParentStatus.Conditions, metav1.Condition{ - Type: string(gwv1.RouteConditionAccepted), - Status: metav1.ConditionFalse, - ObservedGeneration: params.Generation, - LastTransitionTime: metav1.Time{Time: time.Now()}, - Reason: string(gwv1.RouteReasonNoMatchingListenerHostname), - Message: "No matching hostnames were found between the listener and the route.", - }) - } - } - - if metautil.FindStatusCondition(routeParentStatus.Conditions, string(gwv1.RouteConditionResolvedRefs)) == nil { - metautil.SetStatusCondition(&routeParentStatus.Conditions, metav1.Condition{ - Type: string(gwv1.RouteConditionResolvedRefs), - Status: metav1.ConditionTrue, - ObservedGeneration: params.Generation, - LastTransitionTime: metav1.Time{Time: time.Now()}, - Reason: string(gwv1.RouteReasonResolvedRefs), - Message: fmt.Sprintf("References of %s is resolved", params.GVK.Kind), - }) - } - - if metautil.FindStatusCondition(routeParentStatus.Conditions, string(gwv1.RouteConditionAccepted)) == nil { - metautil.SetStatusCondition(&routeParentStatus.Conditions, metav1.Condition{ - Type: string(gwv1.RouteConditionAccepted), - Status: metav1.ConditionTrue, - ObservedGeneration: params.Generation, - LastTransitionTime: metav1.Time{Time: time.Now()}, - Reason: string(gwv1.RouteReasonAccepted), - Message: fmt.Sprintf("%s is Accepted", params.GVK.Kind), - }) - } - - status = append(status, routeParentStatus) - } - } - - return status -} diff --git a/pkg/gateway/status/route/processor.go b/pkg/gateway/status/route/processor.go new file mode 100644 index 000000000..2c9e1dc04 --- /dev/null +++ b/pkg/gateway/status/route/processor.go @@ -0,0 +1,122 @@ +/* + * MIT License + * + * Copyright (c) since 2021, flomesh.io Authors. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package route + +import ( + "context" + "fmt" + + "sigs.k8s.io/controller-runtime/pkg/cache" + + "github.com/flomesh-io/fsm/pkg/gateway/status" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" + + "github.com/flomesh-io/fsm/pkg/constants" + gwutils "github.com/flomesh-io/fsm/pkg/gateway/utils" +) + +// RouteStatusProcessor is responsible for computing the status of a Route +type RouteStatusProcessor struct { + client cache.Cache +} + +func NewRouteStatusProcessor(cache cache.Cache) *RouteStatusProcessor { + return &RouteStatusProcessor{ + client: cache, + } +} + +// Process computes the status of a Route +func (p *RouteStatusProcessor) Process(_ context.Context, updater status.Updater, update status.RouteStatusObject, parentRefs []gwv1.ParentReference) error { + if activeGateways := gwutils.GetActiveGateways(p.client); len(activeGateways) > 0 { + p.computeRouteParentStatus(activeGateways, update, parentRefs) + + updater.Send(status.Update{ + Resource: update.GetResource(), + NamespacedName: update.GetFullName(), + Mutator: update, + }) + } + + return nil +} + +func (p *RouteStatusProcessor) computeRouteParentStatus(activeGateways []*gwv1.Gateway, update status.RouteStatusObject, parentRefs []gwv1.ParentReference) { + for _, gw := range activeGateways { + for _, parentRef := range parentRefs { + if !gwutils.IsRefToGateway(parentRef, client.ObjectKeyFromObject(gw)) { + continue + } + + u := update.StatusUpdateFor(parentRef) + + allowedListeners := gwutils.GetAllowedListeners(p.client, gw, u) + + count := 0 + for _, listener := range allowedListeners { + hostnames := gwutils.GetValidHostnames(listener.Hostname, update.GetHostnames()) + + //if len(hostnames) == 0 { + // continue + //} + + count += len(hostnames) + } + + switch update.GetTypeMeta().GroupVersionKind().Kind { + case constants.GatewayAPIHTTPRouteKind, constants.GatewayAPITLSRouteKind, constants.GatewayAPIGRPCRouteKind: + if count == 0 && !u.ConditionExists(gwv1.RouteConditionAccepted) { + u.AddCondition( + gwv1.RouteConditionAccepted, + metav1.ConditionFalse, + gwv1.RouteReasonNoMatchingListenerHostname, + "No matching hostnames were found between the listener and the route.", + ) + } + } + + if !u.ConditionExists(gwv1.RouteConditionResolvedRefs) { + u.AddCondition( + gwv1.RouteConditionResolvedRefs, + metav1.ConditionTrue, + gwv1.RouteReasonResolvedRefs, + fmt.Sprintf("References of %s is resolved", update.GetTypeMeta().GroupVersionKind().Kind), + ) + } + + if !u.ConditionExists(gwv1.RouteConditionAccepted) { + u.AddCondition( + gwv1.RouteConditionAccepted, + metav1.ConditionTrue, + gwv1.RouteReasonAccepted, + fmt.Sprintf("%s is Accepted", update.GetTypeMeta().GroupVersionKind().Kind), + ) + } + } + } +} diff --git a/pkg/gateway/status/route/route.go b/pkg/gateway/status/route/route.go new file mode 100644 index 000000000..f5d738d1f --- /dev/null +++ b/pkg/gateway/status/route/route.go @@ -0,0 +1,339 @@ +package route + +import ( + "fmt" + "time" + + "github.com/flomesh-io/fsm/pkg/gateway/status" + + metautil "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" + gwv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + + "github.com/flomesh-io/fsm/pkg/constants" +) + +type RouteStatusUpdate struct { + objectMeta *metav1.ObjectMeta + typeMeta *metav1.TypeMeta + routeParentStatuses []*gwv1.RouteParentStatus + hostnames []gwv1.Hostname + resource client.Object + transitionTime metav1.Time + fullName types.NamespacedName + generation int64 +} + +func (r *RouteStatusUpdate) GetObjectMeta() *metav1.ObjectMeta { + return r.objectMeta +} + +func (r *RouteStatusUpdate) GetTypeMeta() *metav1.TypeMeta { + return r.typeMeta +} + +func (r *RouteStatusUpdate) GetRouteParentStatuses() []*gwv1.RouteParentStatus { + return r.routeParentStatuses +} + +func (r *RouteStatusUpdate) GetHostnames() []gwv1.Hostname { + return r.hostnames +} + +func (r *RouteStatusUpdate) GetResource() client.Object { + return r.resource +} + +func (r *RouteStatusUpdate) GetTransitionTime() metav1.Time { + return r.transitionTime +} + +func (r *RouteStatusUpdate) GetFullName() types.NamespacedName { + return r.fullName +} + +func (r *RouteStatusUpdate) GetGeneration() int64 { + return r.generation +} + +func NewRouteStatusUpdate(resource client.Object, meta *metav1.ObjectMeta, typeMeta *metav1.TypeMeta, hostnames []gwv1.Hostname, routeParentStatuses []*gwv1.RouteParentStatus) status.RouteStatusObject { + return &RouteStatusUpdate{ + objectMeta: meta, + typeMeta: typeMeta, + routeParentStatuses: routeParentStatuses, + resource: resource, + hostnames: hostnames, + transitionTime: metav1.Time{Time: time.Now()}, + fullName: types.NamespacedName{Namespace: meta.Namespace, Name: meta.Name}, + generation: meta.Generation, + } +} + +type RouteParentStatusUpdate struct { + *RouteStatusUpdate + ParentRef gwv1.ParentReference +} + +func (r *RouteParentStatusUpdate) GetRouteStatusObject() status.RouteStatusObject { + return r.RouteStatusUpdate +} + +func (r *RouteParentStatusUpdate) GetParentRef() gwv1.ParentReference { + return r.ParentRef +} + +func (r *RouteStatusUpdate) StatusUpdateFor(parentRef gwv1.ParentReference) status.RouteParentStatusObject { + return &RouteParentStatusUpdate{ + RouteStatusUpdate: r, + ParentRef: parentRef, + } +} + +func (r *RouteParentStatusUpdate) AddCondition(conditionType gwv1.RouteConditionType, status metav1.ConditionStatus, reason gwv1.RouteConditionReason, message string) metav1.Condition { + var rps *gwv1.RouteParentStatus + + for _, v := range r.routeParentStatuses { + if v.ParentRef == r.ParentRef { + rps = v + break + } + } + + if rps == nil { + rps = &gwv1.RouteParentStatus{ + ParentRef: r.ParentRef, + ControllerName: constants.GatewayController, + } + + r.routeParentStatuses = append(r.routeParentStatuses, rps) + } + + msg := message + if cond := metautil.FindStatusCondition(rps.Conditions, string(conditionType)); cond != nil { + msg = cond.Message + ", " + message + } + + cond := metav1.Condition{ + Reason: string(reason), + Status: status, + Type: string(conditionType), + Message: msg, + LastTransitionTime: metav1.NewTime(time.Now()), + ObservedGeneration: r.generation, + } + + metautil.SetStatusCondition(&rps.Conditions, cond) + + return cond +} + +func (r *RouteParentStatusUpdate) ConditionExists(conditionType gwv1.RouteConditionType) bool { + for _, c := range r.ConditionsForParentRef(r.ParentRef) { + if c.Type == string(conditionType) { + return true + } + } + return false +} + +func (r *RouteStatusUpdate) ConditionsForParentRef(parentRef gwv1.ParentReference) []metav1.Condition { + for _, rps := range r.routeParentStatuses { + if rps.ParentRef == parentRef { + return rps.Conditions + } + } + + return nil +} + +func (r *RouteStatusUpdate) Mutate(obj client.Object) client.Object { + var newRouteParentStatuses []gwv1.RouteParentStatus + + for _, rps := range r.routeParentStatuses { + for i := range rps.Conditions { + cond := &rps.Conditions[i] + + cond.ObservedGeneration = r.generation + cond.LastTransitionTime = r.transitionTime + } + + newRouteParentStatuses = append(newRouteParentStatuses, *rps) + } + + switch o := obj.(type) { + case *gwv1.HTTPRoute: + route := o.DeepCopy() + + // Get all the RouteParentStatuses that are for other Gateways. + //for _, rps := range o.Status.Parents { + //if !gwutils.IsRefToGateway(rps.ParentRef, r.GatewayRef) { + //newRouteParentStatuses = append(newRouteParentStatuses, rps) + //} + //} + + route.Status.Parents = newRouteParentStatuses + + return route + case *gwv1.GRPCRoute: + route := o.DeepCopy() + + // Get all the RouteParentStatuses that are for other Gateways. + //for _, rps := range o.Status.Parents { + // //if !gwutils.IsRefToGateway(rps.ParentRef, r.GatewayRef) { + // newRouteParentStatuses = append(newRouteParentStatuses, rps) + // //} + //} + + route.Status.Parents = newRouteParentStatuses + + return route + case *gwv1alpha2.TLSRoute: + route := o.DeepCopy() + + // Get all the RouteParentStatuses that are for other Gateways. + //for _, rps := range o.Status.Parents { + // //if !gwutils.IsRefToGateway(rps.ParentRef, r.GatewayRef) { + // newRouteParentStatuses = append(newRouteParentStatuses, rps) + // //} + //} + + route.Status.Parents = newRouteParentStatuses + + return route + case *gwv1alpha2.TCPRoute: + route := o.DeepCopy() + + // Get all the RouteParentStatuses that are for other Gateways. + //for _, rps := range o.Status.Parents { + // //if !gwutils.IsRefToGateway(rps.ParentRef, r.GatewayRef) { + // newRouteParentStatuses = append(newRouteParentStatuses, rps) + // //} + //} + + route.Status.Parents = newRouteParentStatuses + + return route + case *gwv1alpha2.UDPRoute: + route := o.DeepCopy() + + // Get all the RouteParentStatuses that are for other Gateways. + //for _, rps := range o.Status.Parents { + // //if !gwutils.IsRefToGateway(rps.ParentRef, r.GatewayRef) { + // newRouteParentStatuses = append(newRouteParentStatuses, rps) + // //} + //} + + route.Status.Parents = newRouteParentStatuses + + return route + + default: + panic(fmt.Sprintf("Unsupported %T object %s/%s in RouteConditionsUpdate status mutator", obj, r.fullName.Namespace, r.fullName.Name)) + } +} + +type RouteStatusHolder struct { + objectMeta *metav1.ObjectMeta + typeMeta *metav1.TypeMeta + routeParentStatuses []*gwv1.RouteParentStatus + hostnames []gwv1.Hostname + resource client.Object + transitionTime metav1.Time + fullName types.NamespacedName + generation int64 +} + +func (r *RouteStatusHolder) Mutate(obj client.Object) client.Object { + return obj +} + +func (r *RouteStatusHolder) GetObjectMeta() *metav1.ObjectMeta { + return r.objectMeta +} + +func (r *RouteStatusHolder) GetTypeMeta() *metav1.TypeMeta { + return r.typeMeta +} + +func (r *RouteStatusHolder) GetRouteParentStatuses() []*gwv1.RouteParentStatus { + return r.routeParentStatuses +} + +func (r *RouteStatusHolder) GetHostnames() []gwv1.Hostname { + return r.hostnames +} + +func (r *RouteStatusHolder) GetResource() client.Object { + return r.resource +} + +func (r *RouteStatusHolder) GetTransitionTime() metav1.Time { + return r.transitionTime +} + +func (r *RouteStatusHolder) GetFullName() types.NamespacedName { + return r.fullName +} + +func (r *RouteStatusHolder) GetGeneration() int64 { + return r.generation +} + +func (r *RouteStatusHolder) StatusUpdateFor(parentRef gwv1.ParentReference) status.RouteParentStatusObject { + return &RouteParentStatusHolder{ + RouteStatusHolder: r, + ParentRef: parentRef, + } +} + +func NewRouteStatusHolder(resource client.Object, meta *metav1.ObjectMeta, typeMeta *metav1.TypeMeta, hostnames []gwv1.Hostname, routeParentStatuses []*gwv1.RouteParentStatus) status.RouteStatusObject { + return &RouteStatusHolder{ + objectMeta: meta, + typeMeta: typeMeta, + routeParentStatuses: routeParentStatuses, + resource: resource, + hostnames: hostnames, + transitionTime: metav1.Time{Time: time.Now()}, + fullName: types.NamespacedName{Namespace: meta.Namespace, Name: meta.Name}, + generation: meta.Generation, + } +} + +type RouteParentStatusHolder struct { + *RouteStatusHolder + ParentRef gwv1.ParentReference +} + +func (r *RouteParentStatusHolder) AddCondition(_ gwv1.RouteConditionType, _ metav1.ConditionStatus, _ gwv1.RouteConditionReason, _ string) metav1.Condition { + return metav1.Condition{} +} + +func (r *RouteParentStatusHolder) ConditionExists(conditionType gwv1.RouteConditionType) bool { + for _, c := range r.ConditionsForParentRef(r.ParentRef) { + if c.Type == string(conditionType) { + return true + } + } + return false +} + +func (r *RouteParentStatusHolder) ConditionsForParentRef(parentRef gwv1.ParentReference) []metav1.Condition { + for _, rps := range r.routeParentStatuses { + if rps.ParentRef == parentRef { + return rps.Conditions + } + } + + return nil +} + +func (r *RouteParentStatusHolder) GetRouteStatusObject() status.RouteStatusObject { + return r.RouteStatusHolder +} + +func (r *RouteParentStatusHolder) GetParentRef() gwv1.ParentReference { + return r.ParentRef +} diff --git a/pkg/gateway/status/route/types.go b/pkg/gateway/status/route/types.go new file mode 100644 index 000000000..aac9ee742 --- /dev/null +++ b/pkg/gateway/status/route/types.go @@ -0,0 +1,5 @@ +package route + +//var ( +// log = logger.New("fsm-gateway/status/route") +//) diff --git a/pkg/gateway/status/status.go b/pkg/gateway/status/status.go index 7b650d80b..a423d03d5 100644 --- a/pkg/gateway/status/status.go +++ b/pkg/gateway/status/status.go @@ -16,6 +16,8 @@ package status import ( "context" + gwpav1alpha1 "github.com/flomesh-io/fsm/pkg/apis/policyattachment/v1alpha1" + "github.com/rs/zerolog" "github.com/google/go-cmp/cmp" @@ -162,7 +164,17 @@ func (u *UpdateWriter) Send(update Update) { // TCPRoute // UDPRoute // GRPCRoute +// AccessControlPolicy +// RateLimitPolicy +// FaultInjectionPolicy +// SessionStickyPolicy +// CircuitBreakingPolicy +// LoadBalancerPolicy +// HealthCheckPolicy +// RetryPolicy +// UpstreamTLSPolicy +//gocyclo:ignore func isStatusEqual(objA, objB interface{}) bool { opts := cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime") switch a := objA.(type) { @@ -184,6 +196,13 @@ func isStatusEqual(objA, objB interface{}) bool { return true } } + case *gwv1.GRPCRoute: + if b, ok := objB.(*gwv1.GRPCRoute); ok { + if cmp.Equal(a.Status, b.Status, opts) { + return true + } + } + case *gwv1alpha2.TLSRoute: if b, ok := objB.(*gwv1alpha2.TLSRoute); ok { if cmp.Equal(a.Status, b.Status, opts) { @@ -202,8 +221,56 @@ func isStatusEqual(objA, objB interface{}) bool { return true } } - case *gwv1.GRPCRoute: - if b, ok := objB.(*gwv1.GRPCRoute); ok { + case *gwpav1alpha1.AccessControlPolicy: + if b, ok := objB.(*gwpav1alpha1.AccessControlPolicy); ok { + if cmp.Equal(a.Status, b.Status, opts) { + return true + } + } + case *gwpav1alpha1.RateLimitPolicy: + if b, ok := objB.(*gwpav1alpha1.RateLimitPolicy); ok { + if cmp.Equal(a.Status, b.Status, opts) { + return true + } + } + case *gwpav1alpha1.FaultInjectionPolicy: + if b, ok := objB.(*gwpav1alpha1.FaultInjectionPolicy); ok { + if cmp.Equal(a.Status, b.Status, opts) { + return true + } + } + case *gwpav1alpha1.SessionStickyPolicy: + if b, ok := objB.(*gwpav1alpha1.SessionStickyPolicy); ok { + if cmp.Equal(a.Status, b.Status, opts) { + return true + } + } + case *gwpav1alpha1.CircuitBreakingPolicy: + if b, ok := objB.(*gwpav1alpha1.CircuitBreakingPolicy); ok { + if cmp.Equal(a.Status, b.Status, opts) { + return true + } + } + case *gwpav1alpha1.LoadBalancerPolicy: + if b, ok := objB.(*gwpav1alpha1.LoadBalancerPolicy); ok { + if cmp.Equal(a.Status, b.Status, opts) { + return true + } + } + case *gwpav1alpha1.HealthCheckPolicy: + if b, ok := objB.(*gwpav1alpha1.HealthCheckPolicy); ok { + if cmp.Equal(a.Status, b.Status, opts) { + return true + } + } + case *gwpav1alpha1.RetryPolicy: + if b, ok := objB.(*gwpav1alpha1.RetryPolicy); ok { + if cmp.Equal(a.Status, b.Status, opts) { + return true + } + } + case *gwpav1alpha1.UpstreamTLSPolicy: + if b, ok := objB.(*gwpav1alpha1.UpstreamTLSPolicy); ok { if cmp.Equal(a.Status, b.Status, opts) { return true } diff --git a/pkg/gateway/status/types.go b/pkg/gateway/status/types.go index 55924de54..55e1a6d73 100644 --- a/pkg/gateway/status/types.go +++ b/pkg/gateway/status/types.go @@ -1,6 +1,34 @@ // Package status implements utility routines related to the status of the Gateway API resource. package status +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" +) + //var ( -// log = logger.New("fsm-gateway/status") +// log = logger.New("fsm-gateway/routestatus") //) + +type RouteStatusObject interface { + Mutator + GetObjectMeta() *metav1.ObjectMeta + GetTypeMeta() *metav1.TypeMeta + GetRouteParentStatuses() []*gwv1.RouteParentStatus + GetHostnames() []gwv1.Hostname + GetResource() client.Object + GetTransitionTime() metav1.Time + GetFullName() types.NamespacedName + GetGeneration() int64 + StatusUpdateFor(parentRef gwv1.ParentReference) RouteParentStatusObject +} + +type RouteParentStatusObject interface { + GetRouteStatusObject() RouteStatusObject + GetParentRef() gwv1.ParentReference + AddCondition(conditionType gwv1.RouteConditionType, status metav1.ConditionStatus, reason gwv1.RouteConditionReason, message string) metav1.Condition + ConditionExists(conditionType gwv1.RouteConditionType) bool + ConditionsForParentRef(parentRef gwv1.ParentReference) []metav1.Condition +} diff --git a/pkg/gateway/types.go b/pkg/gateway/types.go index e0761dc02..c467c7b77 100644 --- a/pkg/gateway/types.go +++ b/pkg/gateway/types.go @@ -4,36 +4,16 @@ package gateway import ( "time" - "sigs.k8s.io/controller-runtime/pkg/manager" - - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/cache" - "github.com/flomesh-io/fsm/pkg/configurator" gwcache "github.com/flomesh-io/fsm/pkg/gateway/cache" "github.com/flomesh-io/fsm/pkg/messaging" - - "github.com/flomesh-io/fsm/pkg/k8s/informers" ) // client is the type used to represent the Kubernetes client for the gateway.networking.k8s.io API group type client struct { - informers *informers.InformerCollection - kubeClient kubernetes.Interface - msgBroker *messaging.Broker - cfg configurator.Configurator - cache gwcache.Cache -} - -// Controller is the interface for the functionality provided by the resources part of the gateway.networking.k8s.io API group -type Controller interface { - cache.ResourceEventHandler - - // Runnable runs the backend broadcast listener - manager.Runnable - - // LeaderElectionRunnable knows if a Runnable needs to be run in the leader election mode. - manager.LeaderElectionRunnable + msgBroker *messaging.Broker + cfg configurator.Configurator + cache gwcache.Cache } const ( diff --git a/pkg/gateway/types/selectors.go b/pkg/gateway/types/selectors.go new file mode 100644 index 000000000..dab715858 --- /dev/null +++ b/pkg/gateway/types/selectors.go @@ -0,0 +1,153 @@ +package types + +import ( + "fmt" + "strings" + + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/selection" +) + +// OneTermSelector Selector + +// OneTermSelector is a selector that matches fields that have a specific index name +func OneTermSelector(k string) fields.Selector { + return &hasIndex{field: k} +} + +type hasIndex struct { + field string +} + +func (t *hasIndex) Matches(ls fields.Fields) bool { + return ls.Has(t.field) +} + +func (t *hasIndex) Empty() bool { + return false +} + +func (t *hasIndex) RequiresExactMatch(field string) (value string, found bool) { + if t.field == field { + return "", true + } + return "", false +} + +func (t *hasIndex) Transform(fn fields.TransformFunc) (fields.Selector, error) { + field, value, err := fn(t.field, "") + if err != nil { + return nil, err + } + if len(field) == 0 && len(value) == 0 { + return fields.Everything(), nil + } + return &hasIndex{field}, nil +} + +func (t *hasIndex) Requirements() fields.Requirements { + return []fields.Requirement{{ + Field: t.field, + Operator: selection.Equals, + Value: "", + }} +} + +func (t *hasIndex) String() string { + return fmt.Sprintf("%v", t.field) +} + +func (t *hasIndex) DeepCopySelector() fields.Selector { + if t == nil { + return nil + } + out := new(hasIndex) + *out = *t + return out +} + +// OrSelectors Selector represents a logical OR of multiple selectors +func OrSelectors(selectors ...fields.Selector) fields.Selector { + return orTerm(selectors) +} + +type orTerm []fields.Selector + +func (t orTerm) Matches(ls fields.Fields) bool { + for _, q := range t { + if q.Matches(ls) { + return true + } + } + + return false +} + +func (t orTerm) Empty() bool { + if t == nil { + return true + } + if len([]fields.Selector(t)) == 0 { + return true + } + for i := range t { + if !t[i].Empty() { + return false + } + } + return true +} + +func (t orTerm) RequiresExactMatch(field string) (string, bool) { + if t == nil || len([]fields.Selector(t)) == 0 { + return "", false + } + for i := range t { + if value, found := t[i].RequiresExactMatch(field); found { + return value, found + } + } + return "", false +} + +func (t orTerm) Transform(fn fields.TransformFunc) (fields.Selector, error) { + next := make([]fields.Selector, 0, len([]fields.Selector(t))) + for _, s := range []fields.Selector(t) { + n, err := s.Transform(fn) + if err != nil { + return nil, err + } + if !n.Empty() { + next = append(next, n) + } + } + return orTerm(next), nil +} + +func (t orTerm) Requirements() fields.Requirements { + reqs := make([]fields.Requirement, 0, len(t)) + for _, s := range []fields.Selector(t) { + rs := s.Requirements() + reqs = append(reqs, rs...) + } + return reqs +} + +func (t orTerm) String() string { + var terms []string + for _, q := range t { + terms = append(terms, q.String()) + } + return strings.Join(terms, ",") +} + +func (t orTerm) DeepCopySelector() fields.Selector { + if t == nil { + return nil + } + out := make([]fields.Selector, len(t)) + for i := range t { + out[i] = t[i].DeepCopySelector() + } + return orTerm(out) +} diff --git a/pkg/gateway/types/types.go b/pkg/gateway/types/types.go index 5c26a5090..4ab14db87 100644 --- a/pkg/gateway/types/types.go +++ b/pkg/gateway/types/types.go @@ -2,7 +2,9 @@ package types import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/cache" + "sigs.k8s.io/controller-runtime/pkg/manager" + "k8s.io/apimachinery/pkg/runtime/schema" gwv1 "sigs.k8s.io/gateway-api/apis/v1" @@ -13,6 +15,17 @@ var ( log = logger.New("fsm-gateway/types") ) +// Controller is the interface for the functionality provided by the resources part of the gateway.networking.k8s.io API group +type Controller interface { + cache.ResourceEventHandler + + // Runnable runs the backend broadcast listener + manager.Runnable + + // LeaderElectionRunnable knows if a Runnable needs to be run in the leader election mode. + manager.LeaderElectionRunnable +} + // PolicyMatchType is the type used to represent the rate limit policy match type type PolicyMatchType string @@ -53,17 +66,6 @@ func (l *Listener) AllowsKind(gvk schema.GroupVersionKind) bool { return false } -// RouteContext is a wrapper around the Gateway API Route object -type RouteContext struct { - Meta metav1.Object - ParentRefs []gwv1.ParentReference - GVK schema.GroupVersionKind - Generation int64 - Hostnames []gwv1.Hostname - Namespace string - ParentStatus []gwv1.RouteParentStatus -} - // CrossNamespaceFrom is the type used to represent the from part of a cross-namespace reference type CrossNamespaceFrom struct { Group string diff --git a/pkg/gateway/utils/gateway.go b/pkg/gateway/utils/gateway.go new file mode 100644 index 000000000..744d6802f --- /dev/null +++ b/pkg/gateway/utils/gateway.go @@ -0,0 +1,207 @@ +package utils + +import ( + "context" + "fmt" + + "k8s.io/apimachinery/pkg/fields" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/flomesh-io/fsm/pkg/constants" + + corev1 "k8s.io/api/core/v1" + metautil "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/cache" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" + + "github.com/flomesh-io/fsm/pkg/gateway/status" + + gwtypes "github.com/flomesh-io/fsm/pkg/gateway/types" +) + +// IsAcceptedGateway returns true if the gateway is accepted +func IsAcceptedGateway(gateway *gwv1.Gateway) bool { + return metautil.IsStatusConditionTrue(gateway.Status.Conditions, string(gwv1.GatewayConditionAccepted)) +} + +// IsProgrammedGateway returns true if the gateway is programmed +func IsProgrammedGateway(gateway *gwv1.Gateway) bool { + return metautil.IsStatusConditionTrue(gateway.Status.Conditions, string(gwv1.GatewayConditionProgrammed)) +} + +// IsActiveGateway returns true if the gateway is active, it stands for the gateway is accepted, programmed and has a valid listener +func IsActiveGateway(gateway *gwv1.Gateway) bool { + return IsAcceptedGateway(gateway) && IsProgrammedGateway(gateway) +} + +// IsListenerProgrammed returns true if the listener is programmed +func IsListenerProgrammed(listenerStatus gwv1.ListenerStatus) bool { + return metautil.IsStatusConditionTrue(listenerStatus.Conditions, string(gwv1.ListenerConditionProgrammed)) +} + +// IsListenerAccepted returns true if the listener is accepted +func IsListenerAccepted(listenerStatus gwv1.ListenerStatus) bool { + return metautil.IsStatusConditionTrue(listenerStatus.Conditions, string(gwv1.ListenerConditionAccepted)) +} + +// IsListenerConflicted returns true if the listener is conflicted +func IsListenerConflicted(listenerStatus gwv1.ListenerStatus) bool { + return metautil.IsStatusConditionTrue(listenerStatus.Conditions, string(gwv1.ListenerConditionConflicted)) +} + +// IsListenerResolvedRefs returns true if the listener is resolved refs +func IsListenerResolvedRefs(listenerStatus gwv1.ListenerStatus) bool { + return metautil.IsStatusConditionTrue(listenerStatus.Conditions, string(gwv1.ListenerConditionResolvedRefs)) +} + +// IsListenerValid returns true if the listener is valid +func IsListenerValid(s gwv1.ListenerStatus) bool { + return IsListenerAccepted(s) && IsListenerProgrammed(s) && !IsListenerConflicted(s) && IsListenerResolvedRefs(s) +} + +func GetActiveGateways(cache cache.Cache) []*gwv1.Gateway { + classes, err := findFSMGatewayClasses(cache) + if err != nil { + log.Error().Msgf("Failed to find GatewayClass: %v", err) + return nil + } + + gateways := make([]*gwv1.Gateway, 0) + for _, cls := range classes { + list := &gwv1.GatewayList{} + if err := cache.List(context.Background(), list, &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(constants.ClassGatewayIndex, cls.Name), + }); err != nil { + log.Error().Msgf("Failed to list Gateways: %v", err) + continue + } + + gateways = append(gateways, ToSlicePtr(list.Items)...) + } + + return filterActiveGateways(gateways) +} + +// filterActiveGateways returns the active gateways from the list of gateways +func filterActiveGateways(allGateways []*gwv1.Gateway) []*gwv1.Gateway { + gateways := make([]*gwv1.Gateway, 0) + + for _, gw := range allGateways { + if IsActiveGateway(gw) { + gateways = append(gateways, gw) + } + } + + return gateways +} + +// GetValidListenersForGateway returns the valid listeners from the gateway +func GetValidListenersForGateway(gw *gwv1.Gateway) []gwtypes.Listener { + listeners := make(map[gwv1.SectionName]gwv1.Listener) + for _, listener := range gw.Spec.Listeners { + listeners[listener.Name] = listener + } + + validListeners := make([]gwtypes.Listener, 0) + for _, s := range gw.Status.Listeners { + if IsListenerValid(s) { + l, ok := listeners[s.Name] + if !ok { + continue + } + validListeners = append(validListeners, gwtypes.Listener{ + Listener: l, + SupportedKinds: s.SupportedKinds, + }) + } + } + + return validListeners +} + +// GetAllowedListeners returns the allowed listeners +func GetAllowedListeners(client cache.Cache, gw *gwv1.Gateway, u status.RouteParentStatusObject) []gwtypes.Listener { + routeGvk := u.GetRouteStatusObject().GetTypeMeta().GroupVersionKind() + routeNs := u.GetRouteStatusObject().GetObjectMeta().Namespace + parentRef := u.GetParentRef() + validListeners := GetValidListenersForGateway(gw) + + selectedListeners := make([]gwtypes.Listener, 0) + for _, validListener := range validListeners { + if (parentRef.SectionName == nil || *parentRef.SectionName == validListener.Name) && + (parentRef.Port == nil || *parentRef.Port == validListener.Port) { + selectedListeners = append(selectedListeners, validListener) + } + } + + if len(selectedListeners) == 0 { + u.AddCondition( + gwv1.RouteConditionAccepted, + metav1.ConditionFalse, + gwv1.RouteReasonNoMatchingParent, + fmt.Sprintf("No listeners match parent ref %s", types.NamespacedName{Namespace: NamespaceDerefOr(parentRef.Namespace, routeNs), Name: string(parentRef.Name)}), + ) + + return nil + } + + allowedListeners := make([]gwtypes.Listener, 0) + for _, selectedListener := range selectedListeners { + if !selectedListener.AllowsKind(routeGvk) { + continue + } + + // Check if the route is in a namespace that the listener allows. + if !NamespaceMatches(client, selectedListener.AllowedRoutes.Namespaces, gw.Namespace, routeNs) { + continue + } + + allowedListeners = append(allowedListeners, selectedListener) + } + + if len(allowedListeners) == 0 { + u.AddCondition( + gwv1.RouteConditionAccepted, + metav1.ConditionFalse, + gwv1.RouteReasonNotAllowedByListeners, + fmt.Sprintf("No matched listeners of parent ref %s", types.NamespacedName{Namespace: NamespaceDerefOr(parentRef.Namespace, routeNs), Name: string(parentRef.Name)}), + ) + + return nil + } + + return allowedListeners +} + +// NamespaceMatches returns true if the namespace matches +func NamespaceMatches(client cache.Cache, namespaces *gwv1.RouteNamespaces, gatewayNamespace, routeNamespace string) bool { + if namespaces == nil || namespaces.From == nil { + return true + } + + switch *namespaces.From { + case gwv1.NamespacesFromAll: + return true + case gwv1.NamespacesFromSame: + return gatewayNamespace == routeNamespace + case gwv1.NamespacesFromSelector: + namespaceSelector, err := metav1.LabelSelectorAsSelector(namespaces.Selector) + if err != nil { + log.Error().Msgf("failed to convert namespace selector: %v", err) + return false + } + + ns := &corev1.Namespace{} + if err := client.Get(context.Background(), types.NamespacedName{Name: routeNamespace}, ns); err != nil { + log.Error().Msgf("failed to get namespace %s: %v", routeNamespace, err) + return false + } + + return namespaceSelector.Matches(labels.Set(ns.Labels)) + } + + return true +} diff --git a/pkg/gateway/utils/gatewayclass.go b/pkg/gateway/utils/gatewayclass.go new file mode 100644 index 000000000..5b1b130e3 --- /dev/null +++ b/pkg/gateway/utils/gatewayclass.go @@ -0,0 +1,51 @@ +package utils + +import ( + "context" + "fmt" + + "k8s.io/apimachinery/pkg/fields" + + "github.com/flomesh-io/fsm/pkg/constants" + + metautil "k8s.io/apimachinery/pkg/api/meta" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" +) + +// IsAcceptedGatewayClass returns true if the gateway class is accepted +func IsAcceptedGatewayClass(gatewayClass *gwv1.GatewayClass) bool { + return gatewayClass.Spec.ControllerName == constants.GatewayController && + metautil.IsStatusConditionTrue(gatewayClass.Status.Conditions, string(gwv1.GatewayClassConditionStatusAccepted)) +} + +// FindGatewayClassByName returns the GatewayClass with the given name +func FindGatewayClassByName(c cache.Cache, name string) (*gwv1.GatewayClass, error) { + gatewayClass := &gwv1.GatewayClass{} + if err := c.Get(context.Background(), client.ObjectKey{Name: name}, gatewayClass); err != nil { + return nil, fmt.Errorf("failed to get gateway class %s: %s", name, err) + } + + return gatewayClass, nil +} + +// findFSMGatewayClasses returns the effective GatewayClasses for FSM +func findFSMGatewayClasses(c cache.Cache) ([]*gwv1.GatewayClass, error) { + list := &gwv1.GatewayClassList{} + if err := c.List(context.Background(), list, &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(constants.ControllerGatewayClassIndex, constants.GatewayController), + }); err != nil { + return nil, fmt.Errorf("failed to list gateway classes: %s", err) + } + + classes := make([]*gwv1.GatewayClass, 0) + for _, cls := range SortResources(ToSlicePtr(list.Items)) { + cls := cls + if IsAcceptedGatewayClass(cls) { + classes = append(classes, cls) + } + } + + return classes, nil +} diff --git a/pkg/gateway/utils/policies.go b/pkg/gateway/utils/policies.go new file mode 100644 index 000000000..1590b8834 --- /dev/null +++ b/pkg/gateway/utils/policies.go @@ -0,0 +1,456 @@ +package utils + +import ( + "context" + + "k8s.io/apimachinery/pkg/fields" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" + + gwpav1alpha1 "github.com/flomesh-io/fsm/pkg/apis/policyattachment/v1alpha1" +) + +// ---------------------------- Access Control ---------------------------- + +// GetAccessControlsMatchTypePort returns a list of AccessControlPolicy objects that match the given selector +func GetAccessControlsMatchTypePort(cache cache.Cache, selector fields.Selector) []client.Object { + list := &gwpav1alpha1.AccessControlPolicyList{} + if err := cache.List(context.Background(), list, &client.ListOptions{FieldSelector: selector}); err != nil { + return nil + } + + return filterValidPolicies( + toClientObjects(ToSlicePtr(list.Items)), + getGatewayRefGrants(cache), + isAcceptedAccessControlPolicy, + func(policy client.Object) bool { + p := policy.(*gwpav1alpha1.AccessControlPolicy) + return len(p.Spec.Ports) == 0 + }, + accessControlPolicyHasAccessToTargetRef, + ) +} + +// GetAccessControlsMatchTypeHostname returns a list of AccessControlPolicy objects that match the given selector +func GetAccessControlsMatchTypeHostname(cache cache.Cache, selector fields.Selector) []client.Object { + list := &gwpav1alpha1.AccessControlPolicyList{} + if err := cache.List(context.Background(), list, &client.ListOptions{FieldSelector: selector}); err != nil { + return nil + } + + return filterValidPolicies( + toClientObjects(ToSlicePtr(list.Items)), + getHostnameRefGrants(cache), + isAcceptedAccessControlPolicy, + func(policy client.Object) bool { + p := policy.(*gwpav1alpha1.AccessControlPolicy) + return len(p.Spec.Hostnames) == 0 + }, + accessControlPolicyHasAccessToTargetRef, + ) +} + +// GetAccessControlsMatchTypeHTTPRoute returns a list of AccessControlPolicy objects that match the given selector +func GetAccessControlsMatchTypeHTTPRoute(cache cache.Cache, selector fields.Selector) []client.Object { + list := &gwpav1alpha1.AccessControlPolicyList{} + if err := cache.List(context.Background(), list, &client.ListOptions{FieldSelector: selector}); err != nil { + return nil + } + + return filterValidPolicies( + toClientObjects(ToSlicePtr(list.Items)), + getHTTPRouteRefGrants(cache), + isAcceptedAccessControlPolicy, + func(policy client.Object) bool { + p := policy.(*gwpav1alpha1.AccessControlPolicy) + return len(p.Spec.HTTPAccessControls) == 0 + }, + accessControlPolicyHasAccessToTargetRef, + ) +} + +// GetAccessControlsMatchTypeGRPCRoute returns a list of AccessControlPolicy objects that match the given selector +func GetAccessControlsMatchTypeGRPCRoute(cache cache.Cache, selector fields.Selector) []client.Object { + list := &gwpav1alpha1.AccessControlPolicyList{} + if err := cache.List(context.Background(), list, &client.ListOptions{FieldSelector: selector}); err != nil { + return nil + } + + return filterValidPolicies( + toClientObjects(ToSlicePtr(list.Items)), + getGRPCRouteRefGrants(cache), + isAcceptedAccessControlPolicy, + func(policy client.Object) bool { + p := policy.(*gwpav1alpha1.AccessControlPolicy) + return len(p.Spec.GRPCAccessControls) == 0 + }, + accessControlPolicyHasAccessToTargetRef, + ) +} + +func isAcceptedAccessControlPolicy(policy client.Object) bool { + p := policy.(*gwpav1alpha1.AccessControlPolicy) + return IsAcceptedPolicyAttachment(p.Status.Conditions) +} + +func accessControlPolicyHasAccessToTargetRef(policy client.Object, refGrants []*gwv1beta1.ReferenceGrant) bool { + p := policy.(*gwpav1alpha1.AccessControlPolicy) + return HasAccessToTargetRef(p, p.Spec.TargetRef, refGrants) +} + +// ---------------------------- Rate Limit ---------------------------- + +// GetRateLimitsMatchTypePort returns a list of RateLimitPolicy objects that match the given selector +func GetRateLimitsMatchTypePort(cache cache.Cache, selector fields.Selector) []client.Object { + list := &gwpav1alpha1.RateLimitPolicyList{} + if err := cache.List(context.Background(), list, &client.ListOptions{FieldSelector: selector}); err != nil { + return nil + } + + return filterValidPolicies( + toClientObjects(ToSlicePtr(list.Items)), + getGatewayRefGrants(cache), + isAcceptedRateLimitPolicy, + func(policy client.Object) bool { + p := policy.(*gwpav1alpha1.RateLimitPolicy) + return len(p.Spec.Ports) == 0 + }, + rateLimitPolicyHasAccessToTargetRef, + ) +} + +// GetRateLimitsMatchTypeHostname returns a list of RateLimitPolicy objects that match the given selector +func GetRateLimitsMatchTypeHostname(cache cache.Cache, selector fields.Selector) []client.Object { + list := &gwpav1alpha1.RateLimitPolicyList{} + if err := cache.List(context.Background(), list, &client.ListOptions{FieldSelector: selector}); err != nil { + return nil + } + + return filterValidPolicies( + toClientObjects(ToSlicePtr(list.Items)), + getHostnameRefGrants(cache), + isAcceptedRateLimitPolicy, + func(policy client.Object) bool { + p := policy.(*gwpav1alpha1.RateLimitPolicy) + return len(p.Spec.Hostnames) == 0 + }, + rateLimitPolicyHasAccessToTargetRef, + ) +} + +// GetRateLimitsMatchTypeHTTPRoute returns a list of RateLimitPolicy objects that match the given selector +func GetRateLimitsMatchTypeHTTPRoute(cache cache.Cache, selector fields.Selector) []client.Object { + list := &gwpav1alpha1.RateLimitPolicyList{} + if err := cache.List(context.Background(), list, &client.ListOptions{FieldSelector: selector}); err != nil { + return nil + } + + return filterValidPolicies( + toClientObjects(ToSlicePtr(list.Items)), + getHTTPRouteRefGrants(cache), + isAcceptedRateLimitPolicy, + func(policy client.Object) bool { + p := policy.(*gwpav1alpha1.RateLimitPolicy) + return len(p.Spec.HTTPRateLimits) == 0 + }, + rateLimitPolicyHasAccessToTargetRef, + ) +} + +// GetRateLimitsMatchTypeGRPCRoute returns a list of RateLimitPolicy objects that match the given selector +func GetRateLimitsMatchTypeGRPCRoute(cache cache.Cache, selector fields.Selector) []client.Object { + list := &gwpav1alpha1.RateLimitPolicyList{} + if err := cache.List(context.Background(), list, &client.ListOptions{FieldSelector: selector}); err != nil { + return nil + } + + return filterValidPolicies( + toClientObjects(ToSlicePtr(list.Items)), + getGRPCRouteRefGrants(cache), + isAcceptedRateLimitPolicy, + func(policy client.Object) bool { + p := policy.(*gwpav1alpha1.RateLimitPolicy) + return len(p.Spec.GRPCRateLimits) == 0 + }, + rateLimitPolicyHasAccessToTargetRef, + ) +} + +func isAcceptedRateLimitPolicy(policy client.Object) bool { + p := policy.(*gwpav1alpha1.RateLimitPolicy) + return IsAcceptedPolicyAttachment(p.Status.Conditions) +} + +func rateLimitPolicyHasAccessToTargetRef(policy client.Object, refGrants []*gwv1beta1.ReferenceGrant) bool { + p := policy.(*gwpav1alpha1.RateLimitPolicy) + return HasAccessToTargetRef(p, p.Spec.TargetRef, refGrants) +} + +// ---------------------------- Fault Injection ---------------------------- + +// GetFaultInjectionsMatchTypeHostname returns a list of FaultInjectionPolicy objects that match the given selector +func GetFaultInjectionsMatchTypeHostname(cache cache.Cache, selector fields.Selector) []client.Object { + list := &gwpav1alpha1.FaultInjectionPolicyList{} + if err := cache.List(context.Background(), list, &client.ListOptions{FieldSelector: selector}); err != nil { + return nil + } + + return filterValidPolicies( + toClientObjects(ToSlicePtr(list.Items)), + getHostnameRefGrants(cache), + isAcceptedFaultInjectionPolicy, + func(policy client.Object) bool { + p := policy.(*gwpav1alpha1.FaultInjectionPolicy) + return len(p.Spec.Hostnames) == 0 + }, + faultInjectionPolicyHasAccessToTargetRef, + ) +} + +// GetFaultInjectionsMatchTypeHTTPRoute returns a list of FaultInjectionPolicy objects that match the given selector +func GetFaultInjectionsMatchTypeHTTPRoute(cache cache.Cache, selector fields.Selector) []client.Object { + list := &gwpav1alpha1.FaultInjectionPolicyList{} + if err := cache.List(context.Background(), list, &client.ListOptions{FieldSelector: selector}); err != nil { + return nil + } + + return filterValidPolicies( + toClientObjects(ToSlicePtr(list.Items)), + getHTTPRouteRefGrants(cache), + isAcceptedFaultInjectionPolicy, + func(policy client.Object) bool { + p := policy.(*gwpav1alpha1.FaultInjectionPolicy) + return len(p.Spec.HTTPFaultInjections) == 0 + }, + faultInjectionPolicyHasAccessToTargetRef, + ) +} + +// GetFaultInjectionsMatchTypeGRPCRoute returns a list of FaultInjectionPolicy objects that match the given selector +func GetFaultInjectionsMatchTypeGRPCRoute(cache cache.Cache, selector fields.Selector) []client.Object { + list := &gwpav1alpha1.FaultInjectionPolicyList{} + if err := cache.List(context.Background(), list, &client.ListOptions{FieldSelector: selector}); err != nil { + return nil + } + + return filterValidPolicies( + toClientObjects(ToSlicePtr(list.Items)), + getGRPCRouteRefGrants(cache), + isAcceptedFaultInjectionPolicy, + func(policy client.Object) bool { + p := policy.(*gwpav1alpha1.FaultInjectionPolicy) + return len(p.Spec.GRPCFaultInjections) == 0 + }, + faultInjectionPolicyHasAccessToTargetRef, + ) +} + +func isAcceptedFaultInjectionPolicy(policy client.Object) bool { + p := policy.(*gwpav1alpha1.FaultInjectionPolicy) + return IsAcceptedPolicyAttachment(p.Status.Conditions) +} + +func faultInjectionPolicyHasAccessToTargetRef(policy client.Object, refGrants []*gwv1beta1.ReferenceGrant) bool { + p := policy.(*gwpav1alpha1.FaultInjectionPolicy) + return HasAccessToTargetRef(p, p.Spec.TargetRef, refGrants) +} + +// ---------------------------- Session Sticky ---------------------------- + +// GetSessionStickiesMatchTypePort returns a list of SessionStickyPolicy objects that match the given selector +func GetSessionStickies(cache cache.Cache, selector fields.Selector) []client.Object { + list := &gwpav1alpha1.SessionStickyPolicyList{} + if err := cache.List(context.Background(), list, &client.ListOptions{FieldSelector: selector}); err != nil { + return nil + } + + return filterValidPolicies( + toClientObjects(ToSlicePtr(list.Items)), + GetServiceRefGrants(cache), + func(policy client.Object) bool { + p := policy.(*gwpav1alpha1.SessionStickyPolicy) + return IsAcceptedPolicyAttachment(p.Status.Conditions) + }, + func(policy client.Object) bool { + return false + }, + func(policy client.Object, refGrants []*gwv1beta1.ReferenceGrant) bool { + p := policy.(*gwpav1alpha1.SessionStickyPolicy) + return HasAccessToTargetRef(p, p.Spec.TargetRef, refGrants) + }, + ) +} + +// ---------------------------- Circuit Breaking ---------------------------- + +// GetCircuitBreakings returns a list of CircuitBreakingPolicy objects that match the given selector +func GetCircuitBreakings(cache cache.Cache, selector fields.Selector) []client.Object { + list := &gwpav1alpha1.CircuitBreakingPolicyList{} + if err := cache.List(context.Background(), list, &client.ListOptions{FieldSelector: selector}); err != nil { + return nil + } + + return filterValidPolicies( + toClientObjects(ToSlicePtr(list.Items)), + GetServiceRefGrants(cache), + func(policy client.Object) bool { + p := policy.(*gwpav1alpha1.CircuitBreakingPolicy) + return IsAcceptedPolicyAttachment(p.Status.Conditions) + }, + func(policy client.Object) bool { + return false + }, + func(policy client.Object, refGrants []*gwv1beta1.ReferenceGrant) bool { + p := policy.(*gwpav1alpha1.CircuitBreakingPolicy) + return HasAccessToTargetRef(p, p.Spec.TargetRef, refGrants) + }, + ) +} + +// ---------------------------- Health Check ---------------------------- + +// GetHealthChecks returns a list of HealthCheckPolicy objects that match the given selector +func GetHealthChecks(cache cache.Cache, selector fields.Selector) []client.Object { + list := &gwpav1alpha1.HealthCheckPolicyList{} + if err := cache.List(context.Background(), list, &client.ListOptions{FieldSelector: selector}); err != nil { + return nil + } + + return filterValidPolicies( + toClientObjects(ToSlicePtr(list.Items)), + GetServiceRefGrants(cache), + func(policy client.Object) bool { + p := policy.(*gwpav1alpha1.HealthCheckPolicy) + return IsAcceptedPolicyAttachment(p.Status.Conditions) + }, + func(policy client.Object) bool { + return false + }, + func(policy client.Object, refGrants []*gwv1beta1.ReferenceGrant) bool { + p := policy.(*gwpav1alpha1.HealthCheckPolicy) + return HasAccessToTargetRef(p, p.Spec.TargetRef, refGrants) + }, + ) +} + +// ---------------------------- Load Balancer ---------------------------- + +// GetLoadBalancers returns a list of LoadBalancerPolicy objects that match the given selector +func GetLoadBalancers(cache cache.Cache, selector fields.Selector) []client.Object { + list := &gwpav1alpha1.LoadBalancerPolicyList{} + if err := cache.List(context.Background(), list, &client.ListOptions{FieldSelector: selector}); err != nil { + return nil + } + + return filterValidPolicies( + toClientObjects(ToSlicePtr(list.Items)), + GetServiceRefGrants(cache), + func(policy client.Object) bool { + p := policy.(*gwpav1alpha1.LoadBalancerPolicy) + return IsAcceptedPolicyAttachment(p.Status.Conditions) + }, + func(policy client.Object) bool { + return false + }, + func(policy client.Object, refGrants []*gwv1beta1.ReferenceGrant) bool { + p := policy.(*gwpav1alpha1.LoadBalancerPolicy) + return HasAccessToTargetRef(p, p.Spec.TargetRef, refGrants) + }, + ) +} + +// ---------------------------- Retry ---------------------------- + +// GetRetries returns a list of RetryPolicy objects that match the given selector +func GetRetries(cache cache.Cache, selector fields.Selector) []client.Object { + list := &gwpav1alpha1.RetryPolicyList{} + if err := cache.List(context.Background(), list, &client.ListOptions{FieldSelector: selector}); err != nil { + return nil + } + + return filterValidPolicies( + toClientObjects(ToSlicePtr(list.Items)), + GetServiceRefGrants(cache), + func(policy client.Object) bool { + p := policy.(*gwpav1alpha1.RetryPolicy) + return IsAcceptedPolicyAttachment(p.Status.Conditions) + }, + func(policy client.Object) bool { + return false + }, + func(policy client.Object, refGrants []*gwv1beta1.ReferenceGrant) bool { + p := policy.(*gwpav1alpha1.RetryPolicy) + return HasAccessToTargetRef(p, p.Spec.TargetRef, refGrants) + }, + ) +} + +// ---------------------------- Upstream TLS ---------------------------- + +// GetUpStreamTLSes returns a list of UpstreamTLSPolicy objects that match the given selector +func GetUpStreamTLSes(cache cache.Cache, selector fields.Selector) []client.Object { + list := &gwpav1alpha1.UpstreamTLSPolicyList{} + if err := cache.List(context.Background(), list, &client.ListOptions{FieldSelector: selector}); err != nil { + return nil + } + + return filterValidPolicies( + toClientObjects(ToSlicePtr(list.Items)), + GetServiceRefGrants(cache), + func(policy client.Object) bool { + p := policy.(*gwpav1alpha1.UpstreamTLSPolicy) + return IsAcceptedPolicyAttachment(p.Status.Conditions) + }, + func(policy client.Object) bool { + return false + }, + func(policy client.Object, refGrants []*gwv1beta1.ReferenceGrant) bool { + p := policy.(*gwpav1alpha1.UpstreamTLSPolicy) + return HasAccessToTargetRef(p, p.Spec.TargetRef, refGrants) + }, + ) +} + +// ---------------------------- Common methods ---------------------------- + +func toClientObjects[T client.Object](policies []T) []client.Object { + objects := make([]client.Object, 0) + for _, p := range policies { + p := p + objects = append(objects, p) + } + + return objects +} + +type isAcceptedFunc func(policy client.Object) bool +type noDataFunc func(policy client.Object) bool +type hasAccessFunc func(policy client.Object, refGrants []*gwv1beta1.ReferenceGrant) bool + +func filterValidPolicies[T client.Object]( + policies []T, + refGrants []*gwv1beta1.ReferenceGrant, + isAccepted isAcceptedFunc, + noData noDataFunc, + hasAccess hasAccessFunc, +) []client.Object { + validPolicies := make([]client.Object, 0) + for _, p := range policies { + if !isAccepted(p) { + continue + } + + if noData(p) { + continue + } + + if !hasAccess(p, refGrants) { + continue + } + + validPolicies = append(validPolicies, p) + } + + return validPolicies +} diff --git a/pkg/gateway/utils/referencegrants.go b/pkg/gateway/utils/referencegrants.go new file mode 100644 index 000000000..bd1d5987f --- /dev/null +++ b/pkg/gateway/utils/referencegrants.go @@ -0,0 +1,110 @@ +package utils + +import ( + "context" + + gwtypes "github.com/flomesh-io/fsm/pkg/gateway/types" + + "k8s.io/apimachinery/pkg/fields" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" + + "github.com/flomesh-io/fsm/pkg/constants" +) + +// GetGatewayRefGrants returns all ReferenceGrants in the cache that target a Gateway +func getGatewayRefGrants(c cache.Cache) []*gwv1beta1.ReferenceGrant { + gatewayRefGrantList := &gwv1beta1.ReferenceGrantList{} + if err := c.List(context.Background(), gatewayRefGrantList, &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(constants.TargetKindRefGrantIndex, constants.GatewayAPIGatewayKind), + }); err != nil { + return nil + } + + return ToSlicePtr(gatewayRefGrantList.Items) +} + +// GetHostnameRefGrants returns all ReferenceGrants in the cache that target a HTTPRoute or GRPCRoute +func getHostnameRefGrants(c cache.Cache) []*gwv1beta1.ReferenceGrant { + list := &gwv1beta1.ReferenceGrantList{} + if err := c.List(context.Background(), list, &client.ListOptions{ + FieldSelector: gwtypes.OrSelectors( + fields.OneTermEqualSelector(constants.TargetKindRefGrantIndex, constants.GatewayAPIHTTPRouteKind), + fields.OneTermEqualSelector(constants.TargetKindRefGrantIndex, constants.GatewayAPIGRPCRouteKind), + ), + }); err != nil { + return nil + } + + return ToSlicePtr(list.Items) +} + +// GetHTTPRouteRefGrants returns all ReferenceGrants in the cache that target a HTTPRoute +func getHTTPRouteRefGrants(c cache.Cache) []*gwv1beta1.ReferenceGrant { + httpRouteRefGrantList := &gwv1beta1.ReferenceGrantList{} + if err := c.List(context.Background(), httpRouteRefGrantList, &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(constants.TargetKindRefGrantIndex, constants.GatewayAPIHTTPRouteKind), + }); err != nil { + return nil + } + + return ToSlicePtr(httpRouteRefGrantList.Items) +} + +// GetGRPCRouteRefGrants returns all ReferenceGrants in the cache that target a GRPCRoute +func getGRPCRouteRefGrants(c cache.Cache) []*gwv1beta1.ReferenceGrant { + grpcRouteRefGrantList := &gwv1beta1.ReferenceGrantList{} + if err := c.List(context.Background(), grpcRouteRefGrantList, &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(constants.TargetKindRefGrantIndex, constants.GatewayAPIGRPCRouteKind), + }); err != nil { + return nil + } + + return ToSlicePtr(grpcRouteRefGrantList.Items) +} + +// GetServiceRefGrants returns all ReferenceGrants in the cache that target a Kubernetes Service +func GetServiceRefGrants(c cache.Cache) []*gwv1beta1.ReferenceGrant { + list := &gwv1beta1.ReferenceGrantList{} + err := c.List(context.Background(), list, &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(constants.TargetKindRefGrantIndex, constants.KubernetesServiceKind), + }) + if err != nil { + log.Error().Msgf("Failed to list ReferenceGrants: %v", err) + return nil + } + + return ToSlicePtr(list.Items) +} + +// GetSecretRefGrants returns all ReferenceGrants in the cache that target a Kubernetes Secret +func GetSecretRefGrants(c cache.Cache) []*gwv1beta1.ReferenceGrant { + list := &gwv1beta1.ReferenceGrantList{} + err := c.List(context.Background(), list, &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(constants.TargetKindRefGrantIndex, constants.KubernetesSecretKind), + }) + if err != nil { + log.Error().Msgf("Failed to list ReferenceGrants: %v", err) + return nil + } + + return ToSlicePtr(list.Items) +} + +// GetCARefGrants returns all ReferenceGrants in the cache that target a Kubernetes Secret or ConfigMap +func GetCARefGrants(c cache.Cache) []*gwv1beta1.ReferenceGrant { + list := &gwv1beta1.ReferenceGrantList{} + err := c.List(context.Background(), list, &client.ListOptions{ + FieldSelector: gwtypes.OrSelectors( + fields.OneTermEqualSelector(constants.TargetKindRefGrantIndex, constants.KubernetesSecretKind), + fields.OneTermEqualSelector(constants.TargetKindRefGrantIndex, constants.KubernetesConfigMapKind), + ), + }) + if err != nil { + log.Error().Msgf("Failed to list ReferenceGrants: %v", err) + return nil + } + + return ToSlicePtr(list.Items) +} diff --git a/pkg/gateway/utils/utils.go b/pkg/gateway/utils/utils.go index 7fe8df2f9..1858cec6e 100644 --- a/pkg/gateway/utils/utils.go +++ b/pkg/gateway/utils/utils.go @@ -26,18 +26,15 @@ package utils import ( - "fmt" + "sort" "strings" - "time" + + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "github.com/flomesh-io/fsm/pkg/logger" - v1 "k8s.io/client-go/listers/core/v1" - - "k8s.io/apimachinery/pkg/labels" - gwv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" @@ -49,7 +46,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" gwv1 "sigs.k8s.io/gateway-api/apis/v1" - "github.com/flomesh-io/fsm/pkg/apis/gateway" "github.com/flomesh-io/fsm/pkg/constants" gwtypes "github.com/flomesh-io/fsm/pkg/gateway/types" ) @@ -58,8 +54,8 @@ var ( log = logger.New("fsm-gateway/utils") ) -// Namespace returns the namespace if it is not nil, otherwise returns the default namespace -func Namespace(ns *gwv1.Namespace, defaultNs string) string { +// NamespaceDerefOr returns the namespace if it is not nil, otherwise returns the default namespace +func NamespaceDerefOr(ns *gwv1.Namespace, defaultNs string) string { if ns == nil { return defaultNs } @@ -67,57 +63,13 @@ func Namespace(ns *gwv1.Namespace, defaultNs string) string { return string(*ns) } -// IsAcceptedGatewayClass returns true if the gateway class is accepted -func IsAcceptedGatewayClass(gatewayClass *gwv1.GatewayClass) bool { - return metautil.IsStatusConditionTrue(gatewayClass.Status.Conditions, string(gwv1.GatewayClassConditionStatusAccepted)) -} - -// IsActiveGatewayClass returns true if the gateway class is active -func IsActiveGatewayClass(gatewayClass *gwv1.GatewayClass) bool { - return metautil.IsStatusConditionTrue(gatewayClass.Status.Conditions, string(gateway.GatewayClassConditionStatusActive)) -} - -// IsEffectiveGatewayClass returns true if the gateway class is effective -func IsEffectiveGatewayClass(gatewayClass *gwv1.GatewayClass) bool { - return IsAcceptedGatewayClass(gatewayClass) && IsActiveGatewayClass(gatewayClass) -} - -// IsAcceptedGateway returns true if the gateway is accepted -func IsAcceptedGateway(gateway *gwv1.Gateway) bool { - return metautil.IsStatusConditionTrue(gateway.Status.Conditions, string(gwv1.GatewayConditionAccepted)) -} - -// IsActiveGateway returns true if the gateway is active -func IsActiveGateway(gateway *gwv1.Gateway) bool { - hasValidListener := false - - for _, listenerStatus := range gateway.Status.Listeners { - if IsListenerAccepted(listenerStatus) && IsListenerProgrammed(listenerStatus) { - hasValidListener = true - break - } - } - - return IsAcceptedGateway(gateway) && hasValidListener -} - -// IsListenerProgrammed returns true if the listener is programmed -func IsListenerProgrammed(listenerStatus gwv1.ListenerStatus) bool { - return metautil.IsStatusConditionTrue(listenerStatus.Conditions, string(gwv1.ListenerConditionProgrammed)) -} - -// IsListenerAccepted returns true if the listener is accepted -func IsListenerAccepted(listenerStatus gwv1.ListenerStatus) bool { - return metautil.IsStatusConditionTrue(listenerStatus.Conditions, string(gwv1.ListenerConditionAccepted)) -} - // IsAcceptedPolicyAttachment returns true if the policy attachment is accepted func IsAcceptedPolicyAttachment(conditions []metav1.Condition) bool { return metautil.IsStatusConditionTrue(conditions, string(gwv1alpha2.PolicyConditionAccepted)) } // IsRefToGateway returns true if the parent reference is to the gateway -func IsRefToGateway(parentRef gwv1.ParentReference, gateway client.ObjectKey) bool { +func IsRefToGateway(parentRef gwv1.ParentReference, gateway types.NamespacedName) bool { if parentRef.Group != nil && string(*parentRef.Group) != gwv1.GroupName { return false } @@ -133,9 +85,28 @@ func IsRefToGateway(parentRef gwv1.ParentReference, gateway client.ObjectKey) bo return string(parentRef.Name) == gateway.Name } -func HasAccessToTargetRef(policy client.Object, ref gwv1alpha2.NamespacedPolicyTargetReference, referenceGrants []client.Object) bool { +// IsTargetRefToTarget returns true if the target reference is to the target resource +func IsTargetRefToTarget(policy client.Object, targetRef gwv1alpha2.NamespacedPolicyTargetReference, target client.Object) bool { + gvk := target.GetObjectKind().GroupVersionKind() + + if string(targetRef.Group) != gvk.Group { + return false + } + + if string(targetRef.Kind) != gvk.Kind { + return false + } + + if ns := NamespaceDerefOr(targetRef.Namespace, policy.GetNamespace()); ns != target.GetNamespace() { + return false + } + + return string(targetRef.Name) == target.GetName() +} + +// HasAccessToTargetRef returns true if the policy has access to the target reference +func HasAccessToTargetRef(policy client.Object, ref gwv1alpha2.NamespacedPolicyTargetReference, referenceGrants []*gwv1beta1.ReferenceGrant) bool { if ref.Namespace != nil && string(*ref.Namespace) != policy.GetNamespace() && !ValidCrossNamespaceRef( - referenceGrants, gwtypes.CrossNamespaceFrom{ Group: policy.GetObjectKind().GroupVersionKind().Group, Kind: policy.GetObjectKind().GroupVersionKind().Kind, @@ -147,6 +118,7 @@ func HasAccessToTargetRef(policy client.Object, ref gwv1alpha2.NamespacedPolicyT Namespace: string(*ref.Namespace), Name: string(ref.Name), }, + referenceGrants, ) { return false } @@ -154,64 +126,6 @@ func HasAccessToTargetRef(policy client.Object, ref gwv1alpha2.NamespacedPolicyT return true } -// IsRefToTarget returns true if the target reference is to the target object -func IsRefToTarget(referenceGrants []client.Object, policy client.Object, ref gwv1alpha2.NamespacedPolicyTargetReference, target client.Object) bool { - targetGVK := target.GetObjectKind().GroupVersionKind() - - log.Debug().Msgf("[TARGET] IsRefToTarget: policy: %s/%s, ref: %s/%s/%s/%s, target: %s/%s/%s/%s", - policy.GetNamespace(), policy.GetName(), - ref.Group, ref.Kind, Namespace(ref.Namespace, policy.GetNamespace()), ref.Name, - targetGVK.Group, targetGVK.Kind, target.GetNamespace(), target.GetName()) - - if string(ref.Group) != targetGVK.Group { - log.Debug().Msgf("[TARGET] Not refer to the target with the same group, ref.Group: %s, target.Group: %s", ref.Group, targetGVK.Group) - return false - } - - if string(ref.Kind) != targetGVK.Kind { - log.Debug().Msgf("[TARGET] Not refer to the target with the same kind, ref.Kind: %s, target.Kind: %s", ref.Kind, targetGVK.Kind) - return false - } - - // fast-fail, not refer to the target with the same name - if string(ref.Name) != target.GetName() { - log.Debug().Msgf("[TARGET] Not refer to the target with the same name, ref.Name: %s, target.Name: %s", ref.Name, target.GetName()) - return false - } - - if ns := Namespace(ref.Namespace, policy.GetNamespace()); ns != target.GetNamespace() { - log.Debug().Msgf("[TARGET] Not refer to the target with the same namespace, resolved namespace: %s, target.Namespace: %s", ns, target.GetNamespace()) - return false - } - - if ref.Namespace != nil && string(*ref.Namespace) == target.GetNamespace() && string(*ref.Namespace) != policy.GetNamespace() { - log.Debug().Msgf("[TARGET] Found a cross-namespace reference, policy: %s/%s, ref: %s/%s, target: %s/%s", - policy.GetNamespace(), policy.GetName(), string(*ref.Namespace), ref.Name, target.GetNamespace(), target.GetName()) - - policyGVK := policy.GetObjectKind().GroupVersionKind() - result := ValidCrossNamespaceRef( - referenceGrants, - gwtypes.CrossNamespaceFrom{ - Group: policyGVK.Group, - Kind: policyGVK.Kind, - Namespace: policy.GetNamespace(), - }, - gwtypes.CrossNamespaceTo{ - Group: string(ref.Group), - Kind: string(ref.Kind), - Namespace: target.GetNamespace(), - Name: target.GetName(), - }, - ) - - log.Debug().Msgf("[TARGET] Cross-namespace reference result: %v", result) - return result - } - - log.Debug().Msgf("[TARGET] Found a match, ref: %s/%s, target: %s/%s", Namespace(ref.Namespace, policy.GetNamespace()), ref.Name, target.GetNamespace(), target.GetName()) - return true -} - // IsTargetRefToGVK returns true if the target reference is to the given group version kind func IsTargetRefToGVK(targetRef gwv1alpha2.NamespacedPolicyTargetReference, gvk schema.GroupVersionKind) bool { return string(targetRef.Group) == gvk.Group && string(targetRef.Kind) == gvk.Kind @@ -234,93 +148,6 @@ func GroupPointer(group string) *gwv1.Group { return &result } -// GetValidListenersForGateway returns the valid listeners from the gateway -func GetValidListenersForGateway(gw *gwv1.Gateway) []gwtypes.Listener { - listeners := make(map[gwv1.SectionName]gwv1.Listener) - for _, listener := range gw.Spec.Listeners { - listeners[listener.Name] = listener - } - - validListeners := make([]gwtypes.Listener, 0) - for _, status := range gw.Status.Listeners { - if IsListenerAccepted(status) && IsListenerProgrammed(status) { - l, ok := listeners[status.Name] - if !ok { - continue - } - validListeners = append(validListeners, gwtypes.Listener{ - Listener: l, - SupportedKinds: status.SupportedKinds, - }) - } - } - - return validListeners -} - -// GetAllowedListeners returns the allowed listeners -func GetAllowedListeners( - nsLister v1.NamespaceLister, - gw *gwv1.Gateway, - parentRef gwv1.ParentReference, - params *gwtypes.RouteContext, - validListeners []gwtypes.Listener, -) ([]gwtypes.Listener, []metav1.Condition) { - routeGvk := params.GVK - routeGeneration := params.Generation - routeNs := params.Namespace - - selectedListeners := make([]gwtypes.Listener, 0) - invalidListenerConditions := make([]metav1.Condition, 0) - - for _, validListener := range validListeners { - if (parentRef.SectionName == nil || *parentRef.SectionName == validListener.Name) && - (parentRef.Port == nil || *parentRef.Port == validListener.Port) { - selectedListeners = append(selectedListeners, validListener) - } - } - - if len(selectedListeners) == 0 { - invalidListenerConditions = append(invalidListenerConditions, metav1.Condition{ - Type: string(gwv1.RouteConditionAccepted), - Status: metav1.ConditionFalse, - ObservedGeneration: routeGeneration, - LastTransitionTime: metav1.Time{Time: time.Now()}, - Reason: string(gwv1.RouteReasonNoMatchingParent), - Message: fmt.Sprintf("No listeners match parent ref %s", types.NamespacedName{Namespace: Namespace(parentRef.Namespace, routeNs), Name: string(parentRef.Name)}), - }) - return nil, invalidListenerConditions - } - - allowedListeners := make([]gwtypes.Listener, 0) - for _, selectedListener := range selectedListeners { - if !selectedListener.AllowsKind(routeGvk) { - continue - } - - // Check if the route is in a namespace that the listener allows. - if !NamespaceMatches(nsLister, selectedListener.AllowedRoutes.Namespaces, gw.Namespace, routeNs) { - continue - } - - allowedListeners = append(allowedListeners, selectedListener) - } - - if len(allowedListeners) == 0 { - invalidListenerConditions = append(invalidListenerConditions, metav1.Condition{ - Type: string(gwv1.RouteConditionAccepted), - Status: metav1.ConditionFalse, - ObservedGeneration: routeGeneration, - LastTransitionTime: metav1.Time{Time: time.Now()}, - Reason: string(gwv1.RouteReasonNotAllowedByListeners), - Message: fmt.Sprintf("No matched listeners of parent ref %s", types.NamespacedName{Namespace: Namespace(parentRef.Namespace, routeNs), Name: string(parentRef.Name)}), - }) - return nil, invalidListenerConditions - } - - return allowedListeners, invalidListenerConditions -} - // GetValidHostnames returns the valid hostnames func GetValidHostnames(listenerHostname *gwv1.Hostname, routeHostnames []gwv1.Hostname) []string { if len(routeHostnames) == 0 { @@ -367,101 +194,13 @@ func HostnameMatchesWildcardHostname(hostname, wildcardHostname string) bool { return g.Match(hostname) } -// NamespaceMatches returns true if the namespace matches -func NamespaceMatches(nsLister v1.NamespaceLister, namespaces *gwv1.RouteNamespaces, gatewayNamespace, routeNamespace string) bool { - if namespaces == nil || namespaces.From == nil { - return true - } - - switch *namespaces.From { - case gwv1.NamespacesFromAll: - return true - case gwv1.NamespacesFromSame: - return gatewayNamespace == routeNamespace - case gwv1.NamespacesFromSelector: - namespaceSelector, err := metav1.LabelSelectorAsSelector(namespaces.Selector) - if err != nil { - log.Error().Msgf("failed to convert namespace selector: %v", err) - return false - } - - ns, err := nsLister.Get(routeNamespace) - if err != nil { - log.Error().Msgf("failed to get namespace %s: %v", routeNamespace, err) - return false - } - - return namespaceSelector.Matches(labels.Set(ns.Labels)) - } - - return true -} - -func ToRouteContext(route client.Object) *gwtypes.RouteContext { - switch route := route.(type) { - case *gwv1.HTTPRoute: - return &gwtypes.RouteContext{ - Meta: route.GetObjectMeta(), - ParentRefs: route.Spec.ParentRefs, - GVK: route.GroupVersionKind(), - Generation: route.GetGeneration(), - Hostnames: route.Spec.Hostnames, - Namespace: route.GetNamespace(), - ParentStatus: route.Status.Parents, - } - case *gwv1.GRPCRoute: - return &gwtypes.RouteContext{ - Meta: route.GetObjectMeta(), - ParentRefs: route.Spec.ParentRefs, - GVK: route.GroupVersionKind(), - Generation: route.GetGeneration(), - Hostnames: route.Spec.Hostnames, - Namespace: route.GetNamespace(), - ParentStatus: route.Status.Parents, - } - case *gwv1alpha2.TLSRoute: - return &gwtypes.RouteContext{ - Meta: route.GetObjectMeta(), - ParentRefs: route.Spec.ParentRefs, - GVK: route.GroupVersionKind(), - Generation: route.GetGeneration(), - Hostnames: route.Spec.Hostnames, - Namespace: route.GetNamespace(), - ParentStatus: route.Status.Parents, - } - case *gwv1alpha2.TCPRoute: - return &gwtypes.RouteContext{ - Meta: route.GetObjectMeta(), - ParentRefs: route.Spec.ParentRefs, - GVK: route.GroupVersionKind(), - Generation: route.GetGeneration(), - Hostnames: nil, - Namespace: route.GetNamespace(), - ParentStatus: route.Status.Parents, - } - case *gwv1alpha2.UDPRoute: - return &gwtypes.RouteContext{ - Meta: route.GetObjectMeta(), - ParentRefs: route.Spec.ParentRefs, - GVK: route.GroupVersionKind(), - Generation: route.GetGeneration(), - Hostnames: nil, - Namespace: route.GetNamespace(), - ParentStatus: route.Status.Parents, - } - default: - log.Warn().Msgf("Unsupported route type: %T", route) - return nil - } -} - -func ValidCrossNamespaceRef(referenceGrants []client.Object, from gwtypes.CrossNamespaceFrom, to gwtypes.CrossNamespaceTo) bool { +// ValidCrossNamespaceRef returns if the reference is valid across namespaces based on the reference grants +func ValidCrossNamespaceRef(from gwtypes.CrossNamespaceFrom, to gwtypes.CrossNamespaceTo, referenceGrants []*gwv1beta1.ReferenceGrant) bool { if len(referenceGrants) == 0 { return false } - for _, referenceGrant := range referenceGrants { - refGrant := referenceGrant.(*gwv1beta1.ReferenceGrant) + for _, refGrant := range referenceGrants { log.Debug().Msgf("Evaluating ReferenceGrant: %s/%s", refGrant.GetNamespace(), refGrant.GetName()) if refGrant.Namespace != to.Namespace { @@ -505,15 +244,82 @@ func ValidCrossNamespaceRef(referenceGrants []client.Object, from gwtypes.CrossN return false } -func GetActiveGateways(allGateways []client.Object) []*gwv1.Gateway { - gateways := make([]*gwv1.Gateway, 0) - - for _, gw := range allGateways { - gw := gw.(*gwv1.Gateway) - if IsActiveGateway(gw) { - gateways = append(gateways, gw) +// SortResources sorts the resources by creation timestamp and name +func SortResources[T client.Object](resources []T) []T { + sort.Slice(resources, func(i, j int) bool { + if resources[i].GetCreationTimestamp().Time.Equal(resources[j].GetCreationTimestamp().Time) { + return client.ObjectKeyFromObject(resources[i]).String() < client.ObjectKeyFromObject(resources[j]).String() } + + return resources[i].GetCreationTimestamp().Time.Before(resources[j].GetCreationTimestamp().Time) + }) + + return resources +} + +func ToSlicePtr[T any](slice []T) []*T { + ptrs := make([]*T, len(slice)) + for i, v := range slice { + v := v + ptrs[i] = &v } + return ptrs +} - return gateways +// IsValidRefToGroupKindOfCA returns true if the reference is to a ConfigMap or Secret in the core group +func IsValidRefToGroupKindOfCA(ref gwv1.ObjectReference) bool { + if ref.Group != corev1.GroupName { + return false + } + + if ref.Kind == constants.KubernetesSecretKind || ref.Kind == constants.KubernetesConfigMapKind { + return true + } + + return false +} + +// IsValidBackendRefToGroupKindOfService returns true if the reference is to a Service in the core group +func IsValidBackendRefToGroupKindOfService(ref gwv1.BackendObjectReference) bool { + if ref.Group == nil { + return false + } + + if ref.Kind == nil { + return false + } + + if (string(*ref.Kind) == constants.KubernetesServiceKind && string(*ref.Group) == constants.KubernetesCoreGroup) || + (string(*ref.Kind) == constants.FlomeshAPIServiceImportKind && string(*ref.Group) == constants.FlomeshMCSAPIGroup) { + return true + } + + return false +} + +// IsValidRefToGroupKindOfSecret returns true if the reference is to a Secret in the core group +func IsValidRefToGroupKindOfSecret(ref gwv1.SecretObjectReference) bool { + if ref.Group == nil { + return false + } + + if ref.Kind == nil { + return false + } + + if string(*ref.Group) == constants.KubernetesCoreGroup && string(*ref.Kind) == constants.KubernetesSecretKind { + return true + } + + return false +} + +// IsValidTargetRefToGroupKindOfService checks if the target reference is valid to the group kind of service +func IsValidTargetRefToGroupKindOfService(ref gwv1alpha2.NamespacedPolicyTargetReference) bool { + if (ref.Kind == constants.KubernetesServiceKind && ref.Group == constants.KubernetesCoreGroup) || + (ref.Kind == constants.FlomeshAPIServiceImportKind && ref.Group == constants.FlomeshMCSAPIGroup) { + return true + } + + return false } diff --git a/pkg/health/health.go b/pkg/health/health.go index fe5b27bff..95dd54961 100644 --- a/pkg/health/health.go +++ b/pkg/health/health.go @@ -46,10 +46,9 @@ func (httpProbe HTTPProbe) Probe() (int, error) { if httpProbe.Protocol == ProtocolHTTPS { // Certificate validation is to be skipped for HTTPS probes // similar to how k8s api server handles HTTPS probes. - // #nosec G402 transport := &http.Transport{ TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, + InsecureSkipVerify: true, // #nosec G402 MinVersion: tls.VersionTLS13, }, } diff --git a/pkg/ingress/providers/pipy/cache/cache.go b/pkg/ingress/providers/pipy/cache/cache.go index c6ea5094f..d6b41e84c 100644 --- a/pkg/ingress/providers/pipy/cache/cache.go +++ b/pkg/ingress/providers/pipy/cache/cache.go @@ -25,22 +25,26 @@ package cache import ( + "context" "fmt" "strings" "sync" "time" + networkingv1 "k8s.io/api/networking/v1" + + "sigs.k8s.io/controller-runtime/pkg/cache" + + cctx "github.com/flomesh-io/fsm/pkg/context" + mapset "github.com/deckarep/golang-set/v2" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/events" "github.com/flomesh-io/fsm/pkg/configurator" repocfg "github.com/flomesh-io/fsm/pkg/ingress/providers/pipy/route" ingresspipy "github.com/flomesh-io/fsm/pkg/ingress/providers/pipy/utils" - fsminformers "github.com/flomesh-io/fsm/pkg/k8s/informers" "github.com/flomesh-io/fsm/pkg/logger" "github.com/flomesh-io/fsm/pkg/repo" "github.com/flomesh-io/fsm/pkg/utils" @@ -48,10 +52,9 @@ import ( // Cache is the type used to represent the cache for the ingress controller type Cache struct { - kubeClient kubernetes.Interface - recorder events.EventRecorder - cfg configurator.Configurator - informers *fsminformers.InformerCollection + recorder events.EventRecorder + cfg configurator.Configurator + client cache.Cache serviceChanges *ServiceChangeTracker endpointsChanges *EndpointChangeTracker @@ -66,13 +69,6 @@ type Cache struct { mu sync.Mutex - //endpointsSynced bool - //servicesSynced bool - //ingressesSynced bool - //ingressClassesSynced bool - //serviceImportSynced bool - //initialized int32 - repoClient *repo.PipyRepoClient broadcaster events.EventBroadcaster @@ -85,54 +81,34 @@ var ( ) // NewCache creates a new cache for the ingress controller -func NewCache(kubeClient kubernetes.Interface, informers *fsminformers.InformerCollection, cfg configurator.Configurator) *Cache { - eventBroadcaster := events.NewBroadcaster(&events.EventSinkImpl{Interface: kubeClient.EventsV1()}) +func NewCache(ctx *cctx.ControllerContext) *Cache { + eventBroadcaster := events.NewBroadcaster(&events.EventSinkImpl{Interface: ctx.KubeClient.EventsV1()}) recorder := eventBroadcaster.NewRecorder(scheme.Scheme, "fsm-cluster-connector-local") + cfg := ctx.Configurator repoBaseURL := fmt.Sprintf("%s://%s:%d", "http", cfg.GetRepoServerIPAddr(), cfg.GetProxyServerPort()) c := &Cache{ - kubeClient: kubeClient, + client: ctx.Manager.GetCache(), recorder: recorder, - cfg: cfg, - informers: informers, + cfg: ctx.Configurator, serviceMap: make(ServiceMap), serviceImportMap: make(ServiceImportMap), endpointsMap: make(EndpointsMap), ingressMap: make(IngressMap), multiClusterEndpointsMap: make(MultiClusterEndpointsMap), - repoClient: repo.NewRepoClient(repoBaseURL, cfg.GetFSMLogLevel()), + repoClient: repo.NewRepoClient(repoBaseURL, ctx.Configurator.GetFSMLogLevel()), broadcaster: eventBroadcaster, } - c.serviceChanges = NewServiceChangeTracker(enrichServiceInfo, recorder, kubeClient, informers) - c.serviceImportChanges = NewServiceImportChangeTracker(enrichServiceImportInfo, nil, recorder, informers) + client := ctx.Manager.GetCache() + c.serviceChanges = NewServiceChangeTracker(enrichServiceInfo, recorder, client) + c.serviceImportChanges = NewServiceImportChangeTracker(enrichServiceImportInfo, nil, recorder, client) c.endpointsChanges = NewEndpointChangeTracker(nil, recorder) - c.ingressChanges = NewIngressChangeTracker(kubeClient, informers, recorder) + c.ingressChanges = NewIngressChangeTracker(client, recorder) return c } -// -//func (c *Cache) GetBroadcaster() events.EventBroadcaster { -// return c.broadcaster -//} -// -//func (c *Cache) GetRecorder() events.EventRecorder { -// return c.recorder -//} - -//func (c *Cache) setInitialized(value bool) { -// var initialized int32 -// if value { -// initialized = 1 -// } -// atomic.StoreInt32(&c.initialized, initialized) -//} -// -//func (c *Cache) isInitialized() bool { -// return atomic.LoadInt32(&c.initialized) > 0 -//} - // SyncRoutes syncs the routes to the repo func (c *Cache) SyncRoutes() { c.mu.Lock() @@ -209,19 +185,19 @@ func (c *Cache) SyncRoutes() { func (c *Cache) refreshIngress() { log.Info().Msgf("Refreshing Ingress Map ...") - ingresses, err := c.informers.GetListers().K8sIngress. - Ingresses(corev1.NamespaceAll). - List(labels.Everything()) + ingresses := &networkingv1.IngressList{} + err := c.client.List(context.Background(), ingresses) if err != nil { log.Error().Msgf("Failed to list all ingresses: %s", err) } - for _, ing := range ingresses { - if !ingresspipy.IsValidPipyIngress(ing) { + for _, ing := range ingresses.Items { + ing := ing + if !ingresspipy.IsValidPipyIngress(&ing) { continue } - c.ingressChanges.Update(nil, ing) + c.ingressChanges.Update(nil, &ing) } c.ingressMap.Update(c.ingressChanges) diff --git a/pkg/ingress/providers/pipy/cache/ingress.go b/pkg/ingress/providers/pipy/cache/ingress.go index c272acde1..9a8001511 100644 --- a/pkg/ingress/providers/pipy/cache/ingress.go +++ b/pkg/ingress/providers/pipy/cache/ingress.go @@ -32,19 +32,18 @@ import ( "strings" "sync" + "sigs.k8s.io/controller-runtime/pkg/cache" + corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/validation" - "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/events" apiconstants "github.com/flomesh-io/fsm/pkg/apis" "github.com/flomesh-io/fsm/pkg/constants" repocfg "github.com/flomesh-io/fsm/pkg/ingress/providers/pipy/route" ingresspipy "github.com/flomesh-io/fsm/pkg/ingress/providers/pipy/utils" - fsminformers "github.com/flomesh-io/fsm/pkg/k8s/informers" "github.com/flomesh-io/fsm/pkg/utils" ) @@ -177,20 +176,18 @@ type ingressChange struct { // IngressChangeTracker tracks changes to Ingresses type IngressChangeTracker struct { - lock sync.Mutex - items map[types.NamespacedName]*ingressChange - kubeClient kubernetes.Interface - informers *fsminformers.InformerCollection - recorder events.EventRecorder + lock sync.Mutex + items map[types.NamespacedName]*ingressChange + recorder events.EventRecorder + client cache.Cache } // NewIngressChangeTracker creates a new IngressChangeTracker -func NewIngressChangeTracker(kubeClient kubernetes.Interface, informers *fsminformers.InformerCollection, recorder events.EventRecorder) *IngressChangeTracker { +func NewIngressChangeTracker(client cache.Cache, recorder events.EventRecorder) *IngressChangeTracker { return &IngressChangeTracker{ - items: make(map[types.NamespacedName]*ingressChange), - kubeClient: kubeClient, - informers: informers, - recorder: recorder, + items: make(map[types.NamespacedName]*ingressChange), + recorder: recorder, + client: client, } } @@ -373,25 +370,12 @@ func createSvcPortNameInstance(namespace, serviceName, portName string) *Service // svcName in namespace/name format func (t *IngressChangeTracker) findService(namespace string, service *networkingv1.IngressServiceBackend) (*corev1.Service, error) { - svcName := fmt.Sprintf("%s/%s", namespace, service.Name) - - // first, find in local store - svc, exists, err := t.informers.GetByKey(fsminformers.InformerKeyService, svcName) - if err != nil { + svc := &corev1.Service{} + if err := t.client.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: service.Name}, svc); err != nil { return nil, err } - if !exists { - log.Warn().Msgf("no object matching key %q in local store, will try to retrieve it from API server.", svcName) - // if not exists in local, retrieve it from remote API server, this's Plan-B, should seldom happns - svc, err = t.kubeClient.CoreV1().Services(namespace).Get(context.TODO(), service.Name, metav1.GetOptions{}) - if err != nil { - return nil, err - } - log.Info().Msgf("Found service %q from API server.", svcName) - } else { - log.Info().Msgf("Found service %q in local store.", svcName) - } - return svc.(*corev1.Service), nil + + return svc, nil } func (t *IngressChangeTracker) checkoutChanges() []*ingressChange { @@ -617,8 +601,9 @@ func (t *IngressChangeTracker) fetchSSLCert(ing *networkingv1.Ingress, ns, name } log.Info().Msgf("Fetching secret %s/%s ...", ns, name) - secret, err := t.informers.GetListers().Secret.Secrets(ns).Get(name) - + //secret, err := t.informers.GetListers().Secret.Secrets(ns).Get(name) + secret := &corev1.Secret{} + err := t.client.Get(context.TODO(), types.NamespacedName{Namespace: ns, Name: name}, secret) if err != nil { log.Error().Msgf("Failed to get secret %s/%s of Ingress %s/%s: %s", ns, name, ing.Namespace, ing.Name, err) return nil diff --git a/pkg/ingress/providers/pipy/cache/service.go b/pkg/ingress/providers/pipy/cache/service.go index 44a71a457..840b08c97 100644 --- a/pkg/ingress/providers/pipy/cache/service.go +++ b/pkg/ingress/providers/pipy/cache/service.go @@ -31,15 +31,14 @@ import ( "strings" "sync" + "sigs.k8s.io/controller-runtime/pkg/cache" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation" - "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/events" utilcache "k8s.io/kubernetes/pkg/proxy/util" - - fsminformers "github.com/flomesh-io/fsm/pkg/k8s/informers" ) type baseServiceInfo struct { @@ -80,8 +79,7 @@ type ServiceChangeTracker struct { items map[types.NamespacedName]*serviceChange enrichServiceInfo enrichServiceInfoFunc recorder events.EventRecorder - informers *fsminformers.InformerCollection - kubeClient kubernetes.Interface + client cache.Cache } // ServiceMap is a map of ServicePortName to ServicePort @@ -149,13 +147,12 @@ func (t *ServiceChangeTracker) newBaseServiceInfo(port *corev1.ServicePort, serv } // NewServiceChangeTracker creates a new ServiceChangeTracker -func NewServiceChangeTracker(enrichServiceInfo enrichServiceInfoFunc, recorder events.EventRecorder, kubeClient kubernetes.Interface, informers *fsminformers.InformerCollection) *ServiceChangeTracker { +func NewServiceChangeTracker(enrichServiceInfo enrichServiceInfoFunc, recorder events.EventRecorder, client cache.Cache) *ServiceChangeTracker { return &ServiceChangeTracker{ items: make(map[types.NamespacedName]*serviceChange), enrichServiceInfo: enrichServiceInfo, recorder: recorder, - informers: informers, - kubeClient: kubeClient, + client: client, } } @@ -252,25 +249,6 @@ func (t *ServiceChangeTracker) shouldSkipService(svc *corev1.Service) bool { return false } -//func (sct *ServiceChangeTracker) serviceImportExists(svc *corev1.Service) bool { -// _, err := sct.informers.GetListers().ServiceImport. -// ServiceImports(svc.Namespace). -// Get(svc.Name) -// if err != nil { -// if errors.IsNotFound(err) { -// // do nothing, not exists, go ahead and check svc -// log.Info().Msgf("ServiceImport %s/%s doesn't exist", svc.Namespace, svc.Name) -// return false -// } -// -// log.Warn().Msgf("Failed to get ServiceImport %s/%s, %s", svc.Namespace, svc.Name, err) -// -// return false -// } -// -// return true -//} - func (sm *ServiceMap) apply(changes *ServiceChangeTracker) { changes.lock.Lock() defer changes.lock.Unlock() diff --git a/pkg/ingress/providers/pipy/cache/serviceimport.go b/pkg/ingress/providers/pipy/cache/serviceimport.go index e74324239..eff354340 100644 --- a/pkg/ingress/providers/pipy/cache/serviceimport.go +++ b/pkg/ingress/providers/pipy/cache/serviceimport.go @@ -25,10 +25,13 @@ package cache import ( + "context" "fmt" "reflect" "sync" + "sigs.k8s.io/controller-runtime/pkg/cache" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" @@ -37,13 +40,11 @@ import ( utilcache "k8s.io/kubernetes/pkg/proxy/util" mcsv1alpha1 "github.com/flomesh-io/fsm/pkg/apis/multicluster/v1alpha1" - fsminformers "github.com/flomesh-io/fsm/pkg/k8s/informers" ) type baseServiceImportInfo struct { - address string - port int - //portName string + address string + port int protocol corev1.Protocol } @@ -70,8 +71,6 @@ type enrichServiceImportInfoFunc func(port *mcsv1alpha1.ServicePort, svcImp *mcs type serviceImportChange struct { previous ServiceImportMap current ServiceImportMap - //previousEndpoints EndpointsMap - //currentEndpoints EndpointsMap } // ServiceImportChangeTracker tracks changes to ServiceImport objects @@ -82,7 +81,7 @@ type ServiceImportChangeTracker struct { enrichServiceImportInfo enrichServiceImportInfoFunc enrichEndpointInfo enrichMultiClusterEndpointFunc recorder events.EventRecorder - informers *fsminformers.InformerCollection + client cache.Cache } // ServiceImportMap is a map of ServicePortName to ServicePort @@ -124,14 +123,14 @@ func (t *ServiceImportChangeTracker) newBaseServiceInfo(port *mcsv1alpha1.Servic } // NewServiceImportChangeTracker creates a new ServiceImportChangeTracker -func NewServiceImportChangeTracker(enrichServiceImportInfo enrichServiceImportInfoFunc, enrichEndpointInfo enrichMultiClusterEndpointFunc, recorder events.EventRecorder, informers *fsminformers.InformerCollection) *ServiceImportChangeTracker { +func NewServiceImportChangeTracker(enrichServiceImportInfo enrichServiceImportInfoFunc, enrichEndpointInfo enrichMultiClusterEndpointFunc, recorder events.EventRecorder, client cache.Cache) *ServiceImportChangeTracker { return &ServiceImportChangeTracker{ items: make(map[types.NamespacedName]*serviceImportChange), endpointItems: make(map[types.NamespacedName]*multiClusterEndpointsChange), enrichServiceImportInfo: enrichServiceImportInfo, enrichEndpointInfo: enrichEndpointInfo, recorder: recorder, - informers: informers, + client: client, } } @@ -265,7 +264,9 @@ func (t *ServiceImportChangeTracker) serviceImportToServiceMap(svcImp *mcsv1alph } func (t *ServiceImportChangeTracker) serviceExists(svcImp *mcsv1alpha1.ServiceImport) (*corev1.Service, bool) { - svc, err := t.informers.GetListers().Service.Services(svcImp.Namespace).Get(svcImp.Name) + svc := &corev1.Service{} + //svc, err := t.informers.GetListers().Service.Services(svcImp.Namespace).Get(svcImp.Name) + err := t.client.Get(context.Background(), types.NamespacedName{Namespace: svcImp.Namespace, Name: svcImp.Name}, svc) if err != nil { if errors.IsNotFound(err) { return nil, false diff --git a/pkg/ingress/providers/pipy/client.go b/pkg/ingress/providers/pipy/client.go index a458c7942..a67a69985 100644 --- a/pkg/ingress/providers/pipy/client.go +++ b/pkg/ingress/providers/pipy/client.go @@ -4,57 +4,71 @@ import ( "github.com/google/go-cmp/cmp" "github.com/rs/zerolog" "golang.org/x/net/context" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/api/meta" - "k8s.io/client-go/kubernetes" k8scache "k8s.io/client-go/tools/cache" + crClient "sigs.k8s.io/controller-runtime/pkg/client" + + mcsv1alpha1 "github.com/flomesh-io/fsm/pkg/apis/multicluster/v1alpha1" + cctx "github.com/flomesh-io/fsm/pkg/context" "github.com/flomesh-io/fsm/pkg/announcements" - "github.com/flomesh-io/fsm/pkg/certificate" - "github.com/flomesh-io/fsm/pkg/configurator" "github.com/flomesh-io/fsm/pkg/ingress/providers/pipy/cache" "github.com/flomesh-io/fsm/pkg/ingress/providers/pipy/repo" "github.com/flomesh-io/fsm/pkg/k8s" "github.com/flomesh-io/fsm/pkg/k8s/events" fsminformers "github.com/flomesh-io/fsm/pkg/k8s/informers" "github.com/flomesh-io/fsm/pkg/logger" - "github.com/flomesh-io/fsm/pkg/messaging" ) var ( log = logger.New("controller-gatewayapi") ) -// NewIngressController returns a ingress.Controller interface related to functionality provided by the resources in the k8s ingress API group -func NewIngressController(informerCollection *fsminformers.InformerCollection, kubeClient kubernetes.Interface, msgBroker *messaging.Broker, cfg configurator.Configurator, certMgr *certificate.Manager) Controller { - return newClient(informerCollection, kubeClient, msgBroker, cfg, certMgr) +// NewIngressController returns an ingress.Controller interface related to functionality provided by the resources in the k8s ingress API group +func NewIngressController(ctx *cctx.ControllerContext) Controller { + return newClient(ctx) } -func newClient(informerCollection *fsminformers.InformerCollection, kubeClient kubernetes.Interface, msgBroker *messaging.Broker, cfg configurator.Configurator, _ *certificate.Manager) *client { +func newClient(ctx *cctx.ControllerContext) *client { c := &client{ - informers: informerCollection, - kubeClient: kubeClient, - msgBroker: msgBroker, - cfg: cfg, - cache: cache.NewCache(kubeClient, informerCollection, cfg), + msgBroker: ctx.MsgBroker, + cfg: ctx.Configurator, + cache: cache.NewCache(ctx), } // Initialize informers - for _, informerKey := range []fsminformers.InformerKey{ - fsminformers.InformerKeyService, - fsminformers.InformerKeyServiceImport, - fsminformers.InformerKeyEndpoints, - fsminformers.InformerKeySecret, - fsminformers.InformerKeyK8sIngressClass, - fsminformers.InformerKeyK8sIngress, + for informerKey, obj := range map[fsminformers.InformerKey]crClient.Object{ + fsminformers.InformerKeyService: &corev1.Service{}, + fsminformers.InformerKeyServiceImport: &mcsv1alpha1.ServiceImport{}, + fsminformers.InformerKeyEndpoints: &corev1.Endpoints{}, + fsminformers.InformerKeySecret: &corev1.Secret{}, + fsminformers.InformerKeyK8sIngressClass: &networkingv1.IngressClass{}, + fsminformers.InformerKeyK8sIngress: &networkingv1.Ingress{}, } { if eventTypes := getEventTypesByInformerKey(informerKey); eventTypes != nil { - c.informers.AddEventHandler(informerKey, c.getEventHandlerFuncs(eventTypes)) + c.informOnResource(ctx, obj, c.getEventHandlerFuncs(eventTypes)) } } return c } +func (c *client) informOnResource(ctx *cctx.ControllerContext, obj crClient.Object, handler k8scache.ResourceEventHandlerFuncs) { + ch := ctx.Manager.GetCache() + + informer, err := ch.GetInformer(context.Background(), obj) + if err != nil { + panic(err) + } + + _, err = informer.AddEventHandler(handler) + if err != nil { + panic(err) + } +} + func (c *client) getEventHandlerFuncs(eventTypes *k8s.EventTypes) k8scache.ResourceEventHandlerFuncs { return k8scache.ResourceEventHandlerFuncs{ AddFunc: c.onAddFunc(eventTypes), diff --git a/pkg/ingress/providers/pipy/types.go b/pkg/ingress/providers/pipy/types.go index 9cf69e776..1a99a7bb3 100644 --- a/pkg/ingress/providers/pipy/types.go +++ b/pkg/ingress/providers/pipy/types.go @@ -26,22 +26,18 @@ package pipy import ( - "k8s.io/client-go/kubernetes" "sigs.k8s.io/controller-runtime/pkg/manager" "github.com/flomesh-io/fsm/pkg/configurator" "github.com/flomesh-io/fsm/pkg/ingress/providers/pipy/cache" - "github.com/flomesh-io/fsm/pkg/k8s/informers" "github.com/flomesh-io/fsm/pkg/messaging" ) // client is the type used to represent the Kubernetes client for the networking.k8s.io API group type client struct { - informers *informers.InformerCollection - kubeClient kubernetes.Interface - msgBroker *messaging.Broker - cfg configurator.Configurator - cache *cache.Cache + msgBroker *messaging.Broker + cfg configurator.Configurator + cache *cache.Cache } // Controller is the interface for the functionality provided by the resources part of the networking.k8s.io API group diff --git a/pkg/k8s/informers/informers.go b/pkg/k8s/informers/informers.go index 3b4a98022..85e793728 100644 --- a/pkg/k8s/informers/informers.go +++ b/pkg/k8s/informers/informers.go @@ -6,13 +6,8 @@ package informers import ( "errors" - "sort" "testing" - "k8s.io/apimachinery/pkg/runtime/schema" - - "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/flomesh-io/fsm/pkg/version" "github.com/rs/zerolog/log" @@ -61,7 +56,7 @@ func NewInformerCollection(meshName string, stop <-chan struct{}, opts ...Inform ic := &InformerCollection{ meshName: meshName, informers: map[InformerKey]cache.SharedIndexInformer{}, - listers: &Lister{}, + //listers: &Lister{}, } // Execute all of the given options (e.g. set clients, set custom stores, etc.) @@ -104,15 +99,15 @@ func WithKubeClient(kubeClient kubernetes.Interface) InformerCollectionOption { ic.informers[InformerKeyConfigMap] = v1api.ConfigMaps().Informer() ic.informers[InformerKeyNamespaceAll] = v1api.Namespaces().Informer() - ic.listers.Service = v1api.Services().Lister() - ic.listers.Secret = v1api.Secrets().Lister() - ic.listers.ConfigMap = v1api.ConfigMaps().Lister() - ic.listers.Endpoints = v1api.Endpoints().Lister() - ic.listers.Namespace = v1api.Namespaces().Lister() + //ic.listers.Service = v1api.Services().Lister() + //ic.listers.Secret = v1api.Secrets().Lister() + //ic.listers.ConfigMap = v1api.ConfigMaps().Lister() + //ic.listers.Endpoints = v1api.Endpoints().Lister() + //ic.listers.Namespace = v1api.Namespaces().Lister() if version.IsEndpointSliceEnabled(kubeClient) { ic.informers[InformerKeyEndpointSlices] = informerFactory.Discovery().V1().EndpointSlices().Informer() - ic.listers.EndpointSlice = informerFactory.Discovery().V1().EndpointSlices().Lister() + //ic.listers.EndpointSlice = informerFactory.Discovery().V1().EndpointSlices().Lister() } } } @@ -131,14 +126,14 @@ func WithKubeClientWithoutNamespace(kubeClient kubernetes.Interface) InformerCol ic.informers[InformerKeySecret] = v1api.Secrets().Informer() ic.informers[InformerKeyConfigMap] = v1api.ConfigMaps().Informer() - ic.listers.Service = v1api.Services().Lister() - ic.listers.Secret = v1api.Secrets().Lister() - ic.listers.ConfigMap = v1api.ConfigMaps().Lister() - ic.listers.Endpoints = v1api.Endpoints().Lister() + //ic.listers.Service = v1api.Services().Lister() + //ic.listers.Secret = v1api.Secrets().Lister() + //ic.listers.ConfigMap = v1api.ConfigMaps().Lister() + //ic.listers.Endpoints = v1api.Endpoints().Lister() if version.IsEndpointSliceEnabled(kubeClient) { ic.informers[InformerKeyEndpointSlices] = informerFactory.Discovery().V1().EndpointSlices().Informer() - ic.listers.EndpointSlice = informerFactory.Discovery().V1().EndpointSlices().Lister() + //ic.listers.EndpointSlice = informerFactory.Discovery().V1().EndpointSlices().Lister() } } } @@ -228,7 +223,7 @@ func WithMultiClusterClient(multiclusterClient multiclusterClientset.Interface) ic.informers[InformerKeyServiceImport] = informerFactory.Multicluster().V1alpha1().ServiceImports().Informer() ic.informers[InformerKeyGlobalTrafficPolicy] = informerFactory.Multicluster().V1alpha1().GlobalTrafficPolicies().Informer() - ic.listers.ServiceImport = informerFactory.Multicluster().V1alpha1().ServiceImports().Lister() + //ic.listers.ServiceImport = informerFactory.Multicluster().V1alpha1().ServiceImports().Lister() } } @@ -249,13 +244,13 @@ func WithIngressClient(kubeClient kubernetes.Interface, nsigClient nsigClientset ic.informers[InformerKeyK8sIngressClass] = informerFactory.Networking().V1().IngressClasses().Informer() ic.informers[InformerKeyK8sIngress] = informerFactory.Networking().V1().Ingresses().Informer() - ic.listers.K8sIngressClass = informerFactory.Networking().V1().IngressClasses().Lister() - ic.listers.K8sIngress = informerFactory.Networking().V1().Ingresses().Lister() + //ic.listers.K8sIngressClass = informerFactory.Networking().V1().IngressClasses().Lister() + //ic.listers.K8sIngress = informerFactory.Networking().V1().Ingresses().Lister() nsigInformerFactory := nsigInformers.NewSharedInformerFactory(nsigClient, DefaultKubeEventResyncInterval) ic.informers[InformerKeyNamespacedIngress] = nsigInformerFactory.Networking().V1alpha1().NamespacedIngresses().Informer() - ic.listers.NamespacedIngress = nsigInformerFactory.Networking().V1alpha1().NamespacedIngresses().Lister() + //ic.listers.NamespacedIngress = nsigInformerFactory.Networking().V1alpha1().NamespacedIngresses().Lister() } } @@ -274,15 +269,15 @@ func WithPolicyAttachmentClient(policyAttachmentClient policyAttachmentClientset ic.informers[InformerKeyUpstreamTLSPolicy] = informerFactory.Gateway().V1alpha1().UpstreamTLSPolicies().Informer() ic.informers[InformerKeyRetryPolicy] = informerFactory.Gateway().V1alpha1().RetryPolicies().Informer() - ic.listers.RateLimitPolicy = informerFactory.Gateway().V1alpha1().RateLimitPolicies().Lister() - ic.listers.SessionStickyPolicy = informerFactory.Gateway().V1alpha1().SessionStickyPolicies().Lister() - ic.listers.LoadBalancerPolicy = informerFactory.Gateway().V1alpha1().LoadBalancerPolicies().Lister() - ic.listers.CircuitBreakingPolicy = informerFactory.Gateway().V1alpha1().CircuitBreakingPolicies().Lister() - ic.listers.AccessControlPolicy = informerFactory.Gateway().V1alpha1().AccessControlPolicies().Lister() - ic.listers.HealthCheckPolicy = informerFactory.Gateway().V1alpha1().HealthCheckPolicies().Lister() - ic.listers.FaultInjectionPolicy = informerFactory.Gateway().V1alpha1().FaultInjectionPolicies().Lister() - ic.listers.UpstreamTLSPolicy = informerFactory.Gateway().V1alpha1().UpstreamTLSPolicies().Lister() - ic.listers.RetryPolicy = informerFactory.Gateway().V1alpha1().RetryPolicies().Lister() + //ic.listers.RateLimitPolicy = informerFactory.Gateway().V1alpha1().RateLimitPolicies().Lister() + //ic.listers.SessionStickyPolicy = informerFactory.Gateway().V1alpha1().SessionStickyPolicies().Lister() + //ic.listers.LoadBalancerPolicy = informerFactory.Gateway().V1alpha1().LoadBalancerPolicies().Lister() + //ic.listers.CircuitBreakingPolicy = informerFactory.Gateway().V1alpha1().CircuitBreakingPolicies().Lister() + //ic.listers.AccessControlPolicy = informerFactory.Gateway().V1alpha1().AccessControlPolicies().Lister() + //ic.listers.HealthCheckPolicy = informerFactory.Gateway().V1alpha1().HealthCheckPolicies().Lister() + //ic.listers.FaultInjectionPolicy = informerFactory.Gateway().V1alpha1().FaultInjectionPolicies().Lister() + //ic.listers.UpstreamTLSPolicy = informerFactory.Gateway().V1alpha1().UpstreamTLSPolicies().Lister() + //ic.listers.RetryPolicy = informerFactory.Gateway().V1alpha1().RetryPolicies().Lister() } } @@ -300,14 +295,14 @@ func WithGatewayAPIClient(gatewayAPIClient gatewayApiClientset.Interface) Inform ic.informers[InformerKeyGatewayAPIUDPRoute] = informerFactory.Gateway().V1alpha2().UDPRoutes().Informer() ic.informers[InformerKeyGatewayAPIReferenceGrant] = informerFactory.Gateway().V1beta1().ReferenceGrants().Informer() - ic.listers.GatewayClass = informerFactory.Gateway().V1().GatewayClasses().Lister() - ic.listers.Gateway = informerFactory.Gateway().V1().Gateways().Lister() - ic.listers.HTTPRoute = informerFactory.Gateway().V1().HTTPRoutes().Lister() - ic.listers.GRPCRoute = informerFactory.Gateway().V1().GRPCRoutes().Lister() - ic.listers.TLSRoute = informerFactory.Gateway().V1alpha2().TLSRoutes().Lister() - ic.listers.TCPRoute = informerFactory.Gateway().V1alpha2().TCPRoutes().Lister() - ic.listers.UDPRoute = informerFactory.Gateway().V1alpha2().UDPRoutes().Lister() - ic.listers.ReferenceGrant = informerFactory.Gateway().V1beta1().ReferenceGrants().Lister() + //ic.listers.GatewayClass = informerFactory.Gateway().V1().GatewayClasses().Lister() + //ic.listers.Gateway = informerFactory.Gateway().V1().Gateways().Lister() + //ic.listers.HTTPRoute = informerFactory.Gateway().V1().HTTPRoutes().Lister() + //ic.listers.GRPCRoute = informerFactory.Gateway().V1().GRPCRoutes().Lister() + //ic.listers.TLSRoute = informerFactory.Gateway().V1alpha2().TLSRoutes().Lister() + //ic.listers.TCPRoute = informerFactory.Gateway().V1alpha2().TCPRoutes().Lister() + //ic.listers.UDPRoute = informerFactory.Gateway().V1alpha2().UDPRoutes().Lister() + //ic.listers.ReferenceGrant = informerFactory.Gateway().V1beta1().ReferenceGrants().Lister() } } @@ -416,160 +411,160 @@ func (ic *InformerCollection) IsMonitoredNamespace(namespace string) bool { } // GetListers returns the listers for the informers -func (ic *InformerCollection) GetListers() *Lister { - return ic.listers -} +//func (ic *InformerCollection) GetListers() *Lister { +// return ic.listers +//} //gocyclo:ignore -func (ic *InformerCollection) GetGatewayResourcesFromCache(resourceType ResourceType, shouldSort bool) []client.Object { - resources := make([]client.Object, 0) - - switch resourceType { - case GatewayClassesResourceType: - classes, err := ic.listers.GatewayClass.List(selectAll) - if err != nil { - log.Error().Msgf("Failed to get GatewayClasses: %s", err) - return resources - } - resources = setGroupVersionKind(classes, constants.GatewayClassGVK) - case GatewaysResourceType: - gateways, err := ic.listers.Gateway.List(selectAll) - if err != nil { - log.Error().Msgf("Failed to get Gateways: %s", err) - return resources - } - resources = setGroupVersionKind(gateways, constants.GatewayGVK) - case HTTPRoutesResourceType: - routes, err := ic.listers.HTTPRoute.List(selectAll) - if err != nil { - log.Error().Msgf("Failed to get HTTPRoutes: %s", err) - return resources - } - resources = setGroupVersionKind(routes, constants.HTTPRouteGVK) - case GRPCRoutesResourceType: - routes, err := ic.listers.GRPCRoute.List(selectAll) - if err != nil { - log.Error().Msgf("Failed to get GRPCRouts: %s", err) - return resources - } - resources = setGroupVersionKind(routes, constants.GRPCRouteGVK) - case TLSRoutesResourceType: - routes, err := ic.listers.TLSRoute.List(selectAll) - if err != nil { - log.Error().Msgf("Failed to get TLSRoutes: %s", err) - return resources - } - resources = setGroupVersionKind(routes, constants.TLSRouteGVK) - case TCPRoutesResourceType: - routes, err := ic.listers.TCPRoute.List(selectAll) - if err != nil { - log.Error().Msgf("Failed to get TCPRoutes: %s", err) - return resources - } - resources = setGroupVersionKind(routes, constants.TCPRouteGVK) - case UDPRoutesResourceType: - routes, err := ic.listers.UDPRoute.List(selectAll) - if err != nil { - log.Error().Msgf("Failed to get UDPRoutes: %s", err) - return resources - } - resources = setGroupVersionKind(routes, constants.UDPRouteGVK) - case ReferenceGrantResourceType: - grants, err := ic.listers.ReferenceGrant.List(selectAll) - if err != nil { - log.Error().Msgf("Failed to get ReferenceGrants: %s", err) - return resources - } - resources = setGroupVersionKind(grants, constants.ReferenceGrantGVK) - case UpstreamTLSPoliciesResourceType: - policies, err := ic.listers.UpstreamTLSPolicy.List(selectAll) - if err != nil { - log.Error().Msgf("Failed to get UpstreamTLSPolicies: %s", err) - return resources - } - resources = setGroupVersionKind(policies, constants.UpstreamTLSPolicyGVK) - case RateLimitPoliciesResourceType: - policies, err := ic.listers.RateLimitPolicy.List(selectAll) - if err != nil { - log.Error().Msgf("Failed to get RateLimitPolicies: %s", err) - return resources - } - resources = setGroupVersionKind(policies, constants.RateLimitPolicyGVK) - case AccessControlPoliciesResourceType: - policies, err := ic.listers.AccessControlPolicy.List(selectAll) - if err != nil { - log.Error().Msgf("Failed to get AccessControlPolicies: %s", err) - return resources - } - resources = setGroupVersionKind(policies, constants.AccessControlPolicyGVK) - case FaultInjectionPoliciesResourceType: - policies, err := ic.listers.FaultInjectionPolicy.List(selectAll) - if err != nil { - log.Error().Msgf("Failed to get FaultInjectionPolicies: %s", err) - return resources - } - resources = setGroupVersionKind(policies, constants.FaultInjectionPolicyGVK) - case SessionStickyPoliciesResourceType: - policies, err := ic.listers.SessionStickyPolicy.List(selectAll) - if err != nil { - log.Error().Msgf("Failed to get SessionStickyPolicies: %s", err) - return resources - } - resources = setGroupVersionKind(policies, constants.SessionStickyPolicyGVK) - case LoadBalancerPoliciesResourceType: - policies, err := ic.listers.LoadBalancerPolicy.List(selectAll) - if err != nil { - log.Error().Msgf("Failed to get LoadBalancerPolicies: %s", err) - return resources - } - resources = setGroupVersionKind(policies, constants.LoadBalancerPolicyGVK) - case CircuitBreakingPoliciesResourceType: - policies, err := ic.listers.CircuitBreakingPolicy.List(selectAll) - if err != nil { - log.Error().Msgf("Failed to get CircuitBreakingPolicies: %s", err) - return resources - } - resources = setGroupVersionKind(policies, constants.CircuitBreakingPolicyGVK) - case HealthCheckPoliciesResourceType: - policies, err := ic.listers.HealthCheckPolicy.List(selectAll) - if err != nil { - log.Error().Msgf("Failed to get HealthCheckPolicies: %s", err) - return resources - } - resources = setGroupVersionKind(policies, constants.HealthCheckPolicyGVK) - case RetryPoliciesResourceType: - policies, err := ic.listers.RetryPolicy.List(selectAll) - if err != nil { - log.Error().Msgf("Failed to get RetryPolicies: %s", err) - return resources - } - resources = setGroupVersionKind(policies, constants.RetryPolicyGVK) - default: - log.Error().Msgf("Unknown resource type: %s", resourceType) - return nil - } - - if shouldSort { - sort.Slice(resources, func(i, j int) bool { - if resources[i].GetCreationTimestamp().Time.Equal(resources[j].GetCreationTimestamp().Time) { - return client.ObjectKeyFromObject(resources[i]).String() < client.ObjectKeyFromObject(resources[j]).String() - } - - return resources[i].GetCreationTimestamp().Time.Before(resources[j].GetCreationTimestamp().Time) - }) - } - - return resources -} - -func setGroupVersionKind[T GatewayAPIResource](objects []T, gvk schema.GroupVersionKind) []client.Object { - resources := make([]client.Object, 0) - - for _, obj := range objects { - obj := client.Object(obj) - obj.GetObjectKind().SetGroupVersionKind(gvk) - resources = append(resources, obj) - } - - return resources -} +//func (ic *InformerCollection) GetGatewayResourcesFromCache(resourceType ResourceType, shouldSort bool) []client.Object { +// resources := make([]client.Object, 0) +// +// switch resourceType { +// case GatewayClassesResourceType: +// classes, err := ic.listers.GatewayClass.List(selectAll) +// if err != nil { +// log.Error().Msgf("Failed to get GatewayClasses: %s", err) +// return resources +// } +// resources = setGroupVersionKind(classes, constants.GatewayClassGVK) +// case GatewaysResourceType: +// gateways, err := ic.listers.Gateway.List(selectAll) +// if err != nil { +// log.Error().Msgf("Failed to get Gateways: %s", err) +// return resources +// } +// resources = setGroupVersionKind(gateways, constants.GatewayGVK) +// case HTTPRoutesResourceType: +// routes, err := ic.listers.HTTPRoute.List(selectAll) +// if err != nil { +// log.Error().Msgf("Failed to get HTTPRoutes: %s", err) +// return resources +// } +// resources = setGroupVersionKind(routes, constants.HTTPRouteGVK) +// case GRPCRoutesResourceType: +// routes, err := ic.listers.GRPCRoute.List(selectAll) +// if err != nil { +// log.Error().Msgf("Failed to get GRPCRouts: %s", err) +// return resources +// } +// resources = setGroupVersionKind(routes, constants.GRPCRouteGVK) +// case TLSRoutesResourceType: +// routes, err := ic.listers.TLSRoute.List(selectAll) +// if err != nil { +// log.Error().Msgf("Failed to get TLSRoutes: %s", err) +// return resources +// } +// resources = setGroupVersionKind(routes, constants.TLSRouteGVK) +// case TCPRoutesResourceType: +// routes, err := ic.listers.TCPRoute.List(selectAll) +// if err != nil { +// log.Error().Msgf("Failed to get TCPRoutes: %s", err) +// return resources +// } +// resources = setGroupVersionKind(routes, constants.TCPRouteGVK) +// case UDPRoutesResourceType: +// routes, err := ic.listers.UDPRoute.List(selectAll) +// if err != nil { +// log.Error().Msgf("Failed to get UDPRoutes: %s", err) +// return resources +// } +// resources = setGroupVersionKind(routes, constants.UDPRouteGVK) +// case ReferenceGrantResourceType: +// grants, err := ic.listers.ReferenceGrant.List(selectAll) +// if err != nil { +// log.Error().Msgf("Failed to get ReferenceGrants: %s", err) +// return resources +// } +// resources = setGroupVersionKind(grants, constants.ReferenceGrantGVK) +// case UpstreamTLSPoliciesResourceType: +// policies, err := ic.listers.UpstreamTLSPolicy.List(selectAll) +// if err != nil { +// log.Error().Msgf("Failed to get UpstreamTLSPolicies: %s", err) +// return resources +// } +// resources = setGroupVersionKind(policies, constants.UpstreamTLSPolicyGVK) +// case RateLimitPoliciesResourceType: +// policies, err := ic.listers.RateLimitPolicy.List(selectAll) +// if err != nil { +// log.Error().Msgf("Failed to get RateLimitPolicies: %s", err) +// return resources +// } +// resources = setGroupVersionKind(policies, constants.RateLimitPolicyGVK) +// case AccessControlPoliciesResourceType: +// policies, err := ic.listers.AccessControlPolicy.List(selectAll) +// if err != nil { +// log.Error().Msgf("Failed to get AccessControlPolicies: %s", err) +// return resources +// } +// resources = setGroupVersionKind(policies, constants.AccessControlPolicyGVK) +// case FaultInjectionPoliciesResourceType: +// policies, err := ic.listers.FaultInjectionPolicy.List(selectAll) +// if err != nil { +// log.Error().Msgf("Failed to get FaultInjectionPolicies: %s", err) +// return resources +// } +// resources = setGroupVersionKind(policies, constants.FaultInjectionPolicyGVK) +// case SessionStickyPoliciesResourceType: +// policies, err := ic.listers.SessionStickyPolicy.List(selectAll) +// if err != nil { +// log.Error().Msgf("Failed to get SessionStickyPolicies: %s", err) +// return resources +// } +// resources = setGroupVersionKind(policies, constants.SessionStickyPolicyGVK) +// case LoadBalancerPoliciesResourceType: +// policies, err := ic.listers.LoadBalancerPolicy.List(selectAll) +// if err != nil { +// log.Error().Msgf("Failed to get LoadBalancerPolicies: %s", err) +// return resources +// } +// resources = setGroupVersionKind(policies, constants.LoadBalancerPolicyGVK) +// case CircuitBreakingPoliciesResourceType: +// policies, err := ic.listers.CircuitBreakingPolicy.List(selectAll) +// if err != nil { +// log.Error().Msgf("Failed to get CircuitBreakingPolicies: %s", err) +// return resources +// } +// resources = setGroupVersionKind(policies, constants.CircuitBreakingPolicyGVK) +// case HealthCheckPoliciesResourceType: +// policies, err := ic.listers.HealthCheckPolicy.List(selectAll) +// if err != nil { +// log.Error().Msgf("Failed to get HealthCheckPolicies: %s", err) +// return resources +// } +// resources = setGroupVersionKind(policies, constants.HealthCheckPolicyGVK) +// case RetryPoliciesResourceType: +// policies, err := ic.listers.RetryPolicy.List(selectAll) +// if err != nil { +// log.Error().Msgf("Failed to get RetryPolicies: %s", err) +// return resources +// } +// resources = setGroupVersionKind(policies, constants.RetryPolicyGVK) +// default: +// log.Error().Msgf("Unknown resource type: %s", resourceType) +// return nil +// } +// +// if shouldSort { +// sort.Slice(resources, func(i, j int) bool { +// if resources[i].GetCreationTimestamp().Time.Equal(resources[j].GetCreationTimestamp().Time) { +// return client.ObjectKeyFromObject(resources[i]).String() < client.ObjectKeyFromObject(resources[j]).String() +// } +// +// return resources[i].GetCreationTimestamp().Time.Before(resources[j].GetCreationTimestamp().Time) +// }) +// } +// +// return resources +//} +// +//func setGroupVersionKind[T GatewayAPIResource](objects []T, gvk schema.GroupVersionKind) []client.Object { +// resources := make([]client.Object, 0) +// +// for _, obj := range objects { +// obj := client.Object(obj) +// obj.GetObjectKind().SetGroupVersionKind(gvk) +// resources = append(resources, obj) +// } +// +// return resources +//} diff --git a/pkg/k8s/informers/types.go b/pkg/k8s/informers/types.go index 6c6199245..0d04be356 100644 --- a/pkg/k8s/informers/types.go +++ b/pkg/k8s/informers/types.go @@ -4,25 +4,6 @@ import ( "errors" "time" - "k8s.io/apimachinery/pkg/labels" - - gwv1 "sigs.k8s.io/gateway-api/apis/v1" - gwv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" - gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" - - gwpav1alpha1lister "github.com/flomesh-io/fsm/pkg/gen/client/policyattachment/listers/policyattachment/v1alpha1" - - v1 "k8s.io/client-go/listers/core/v1" - discoveryv1 "k8s.io/client-go/listers/discovery/v1" - networkingv1 "k8s.io/client-go/listers/networking/v1" - gwv1lister "sigs.k8s.io/gateway-api/pkg/client/listers/apis/v1" - gwv1alpha2lister "sigs.k8s.io/gateway-api/pkg/client/listers/apis/v1alpha2" - gwv1beta1lister "sigs.k8s.io/gateway-api/pkg/client/listers/apis/v1beta1" - - gwpav1alpha1 "github.com/flomesh-io/fsm/pkg/apis/policyattachment/v1alpha1" - mcsv1alpha1 "github.com/flomesh-io/fsm/pkg/gen/client/multicluster/listers/multicluster/v1alpha1" - nsigv1alpha1 "github.com/flomesh-io/fsm/pkg/gen/client/namespacedingress/listers/namespacedingress/v1alpha1" - "k8s.io/client-go/tools/cache" ) @@ -172,40 +153,40 @@ var ( // type should only be passed around as a pointer type InformerCollection struct { informers map[InformerKey]cache.SharedIndexInformer - listers *Lister - meshName string + //listers *Lister + meshName string } // Lister is the listers for the informers in the collection -type Lister struct { - Service v1.ServiceLister - ServiceImport mcsv1alpha1.ServiceImportLister - Endpoints v1.EndpointsLister - EndpointSlice discoveryv1.EndpointSliceLister - Secret v1.SecretLister - ConfigMap v1.ConfigMapLister - GatewayClass gwv1lister.GatewayClassLister - Gateway gwv1lister.GatewayLister - HTTPRoute gwv1lister.HTTPRouteLister - GRPCRoute gwv1lister.GRPCRouteLister - TLSRoute gwv1alpha2lister.TLSRouteLister - TCPRoute gwv1alpha2lister.TCPRouteLister - UDPRoute gwv1alpha2lister.UDPRouteLister - K8sIngressClass networkingv1.IngressClassLister - K8sIngress networkingv1.IngressLister - NamespacedIngress nsigv1alpha1.NamespacedIngressLister - RateLimitPolicy gwpav1alpha1lister.RateLimitPolicyLister - SessionStickyPolicy gwpav1alpha1lister.SessionStickyPolicyLister - LoadBalancerPolicy gwpav1alpha1lister.LoadBalancerPolicyLister - CircuitBreakingPolicy gwpav1alpha1lister.CircuitBreakingPolicyLister - AccessControlPolicy gwpav1alpha1lister.AccessControlPolicyLister - HealthCheckPolicy gwpav1alpha1lister.HealthCheckPolicyLister - FaultInjectionPolicy gwpav1alpha1lister.FaultInjectionPolicyLister - UpstreamTLSPolicy gwpav1alpha1lister.UpstreamTLSPolicyLister - RetryPolicy gwpav1alpha1lister.RetryPolicyLister - ReferenceGrant gwv1beta1lister.ReferenceGrantLister - Namespace v1.NamespaceLister -} +//type Lister struct { +// Service v1.ServiceLister +// ServiceImport mcsv1alpha1.ServiceImportLister +// Endpoints v1.EndpointsLister +// EndpointSlice discoveryv1.EndpointSliceLister +// Secret v1.SecretLister +// ConfigMap v1.ConfigMapLister +// GatewayClass gwv1lister.GatewayClassLister +// Gateway gwv1lister.GatewayLister +// HTTPRoute gwv1lister.HTTPRouteLister +// GRPCRoute gwv1lister.GRPCRouteLister +// TLSRoute gwv1alpha2lister.TLSRouteLister +// TCPRoute gwv1alpha2lister.TCPRouteLister +// UDPRoute gwv1alpha2lister.UDPRouteLister +// K8sIngressClass networkingv1.IngressClassLister +// K8sIngress networkingv1.IngressLister +// NamespacedIngress nsigv1alpha1.NamespacedIngressLister +// RateLimitPolicy gwpav1alpha1lister.RateLimitPolicyLister +// SessionStickyPolicy gwpav1alpha1lister.SessionStickyPolicyLister +// LoadBalancerPolicy gwpav1alpha1lister.LoadBalancerPolicyLister +// CircuitBreakingPolicy gwpav1alpha1lister.CircuitBreakingPolicyLister +// AccessControlPolicy gwpav1alpha1lister.AccessControlPolicyLister +// HealthCheckPolicy gwpav1alpha1lister.HealthCheckPolicyLister +// FaultInjectionPolicy gwpav1alpha1lister.FaultInjectionPolicyLister +// UpstreamTLSPolicy gwpav1alpha1lister.UpstreamTLSPolicyLister +// RetryPolicy gwpav1alpha1lister.RetryPolicyLister +// ReferenceGrant gwv1beta1lister.ReferenceGrantLister +// Namespace v1.NamespaceLister +//} // ResourceType is the type used to represent the type of resource type ResourceType string @@ -282,14 +263,14 @@ const ( ) // GatewayAPIResource is the type used to represent the Gateway API resource -type GatewayAPIResource interface { - *gwv1.GatewayClass | *gwv1.Gateway | - *gwv1.HTTPRoute | *gwv1.GRPCRoute | *gwv1alpha2.TLSRoute | *gwv1alpha2.TCPRoute | *gwv1alpha2.UDPRoute | *gwv1beta1.ReferenceGrant | - *gwpav1alpha1.RateLimitPolicy | *gwpav1alpha1.SessionStickyPolicy | *gwpav1alpha1.LoadBalancerPolicy | - *gwpav1alpha1.CircuitBreakingPolicy | *gwpav1alpha1.AccessControlPolicy | *gwpav1alpha1.HealthCheckPolicy | - *gwpav1alpha1.FaultInjectionPolicy | *gwpav1alpha1.UpstreamTLSPolicy | *gwpav1alpha1.RetryPolicy -} - -var ( - selectAll = labels.Set{}.AsSelector() -) +//type GatewayAPIResource interface { +// *gwv1.GatewayClass | *gwv1.Gateway | +// *gwv1.HTTPRoute | *gwv1.GRPCRoute | *gwv1alpha2.TLSRoute | *gwv1alpha2.TCPRoute | *gwv1alpha2.UDPRoute | *gwv1beta1.ReferenceGrant | +// *gwpav1alpha1.RateLimitPolicy | *gwpav1alpha1.SessionStickyPolicy | *gwpav1alpha1.LoadBalancerPolicy | +// *gwpav1alpha1.CircuitBreakingPolicy | *gwpav1alpha1.AccessControlPolicy | *gwpav1alpha1.HealthCheckPolicy | +// *gwpav1alpha1.FaultInjectionPolicy | *gwpav1alpha1.UpstreamTLSPolicy | *gwpav1alpha1.RetryPolicy +//} + +//var ( +// selectAll = labels.Set{}.AsSelector() +//) diff --git a/pkg/manager/reconciler/controllers.go b/pkg/manager/reconciler/controllers.go index 0a63f729c..c99d40811 100644 --- a/pkg/manager/reconciler/controllers.go +++ b/pkg/manager/reconciler/controllers.go @@ -3,8 +3,6 @@ package reconciler import ( "context" - gatewayApiClientset "sigs.k8s.io/gateway-api/pkg/client/clientset/versioned" - "github.com/flomesh-io/fsm/pkg/gateway/status" fctx "github.com/flomesh-io/fsm/pkg/context" @@ -26,7 +24,7 @@ func RegisterControllers(ctx context.Context) error { kubeClient := cctx.KubeClient if mc.IsIngressEnabled() { - ingressController := pipy.NewIngressController(cctx.InformerCollection, kubeClient, cctx.MsgBroker, mc, cctx.CertManager) + ingressController := pipy.NewIngressController(cctx) if err := mgr.Add(ingressController); err != nil { events.GenericEventRecorder().FatalEvent(err, events.InitializationError, "Error add Ingress Controller to manager") @@ -42,12 +40,7 @@ func RegisterControllers(ctx context.Context) error { return err } - gatewayAPIClient, err := gatewayApiClientset.NewForConfig(cctx.KubeConfig) - if err != nil { - return err - } - - gatewayController := gateway.NewGatewayAPIController(cctx.InformerCollection, kubeClient, gatewayAPIClient, cctx.MsgBroker, mc, cctx.MeshName, cctx.FSMVersion) + gatewayController := gateway.NewGatewayAPIController(cctx) cctx.GatewayEventHandler = gatewayController if err := mgr.Add(gatewayController); err != nil { events.GenericEventRecorder().FatalEvent(err, events.InitializationError, "Error add Gateway Controller to manager") diff --git a/pkg/manager/repo/repo.go b/pkg/manager/repo/repo.go index ee723d8fa..1a2e00f88 100644 --- a/pkg/manager/repo/repo.go +++ b/pkg/manager/repo/repo.go @@ -215,10 +215,10 @@ func visit(files *[]string) filepath.WalkFunc { //} func (r *rebuilder) rebuildRepoJob() error { - log.Debug().Msg("<<<<<< rebuilding repo - start >>>>>> ") + log.Trace().Msg("<<<<<< rebuilding repo - start >>>>>> ") if !r.repoClient.IsRepoUp() { - log.Debug().Msg("Repo is not up, sleeping ...") + log.Trace().Msg("Repo is not up, sleeping ...") return nil } @@ -298,7 +298,7 @@ func (r *rebuilder) rebuildRepoJob() error { for _, gw := range gatewayList.Items { gw := gw // fix lint GO-LOOP-REF if gwutils.IsActiveGateway(&gw) { - gwPath := utils.GatewayCodebasePath(gw.Namespace) + gwPath := utils.GatewayCodebasePath(gw.Namespace, gw.Name) parentPath := utils.GetDefaultGatewaysPath() if err := r.repoClient.DeriveCodebase(gwPath, parentPath); err != nil { return err @@ -307,6 +307,6 @@ func (r *rebuilder) rebuildRepoJob() error { } } - log.Debug().Msg("<<<<<< rebuilding repo - end >>>>>> ") + log.Trace().Msg("<<<<<< rebuilding repo - end >>>>>> ") return nil } diff --git a/pkg/mcs/event/events.go b/pkg/mcs/event/events.go index b17047d75..7fe2fba04 100644 --- a/pkg/mcs/event/events.go +++ b/pkg/mcs/event/events.go @@ -38,7 +38,7 @@ type ServiceExportEvent struct { ServiceExport *mcsv1alpha1.ServiceExport Service *corev1.Service Error string - //Data map[string]interface{} + //data map[string]interface{} } // ClusterKey returns the cluster key diff --git a/pkg/mcs/remote/connector.go b/pkg/mcs/remote/connector.go index 17d396010..7834707f4 100644 --- a/pkg/mcs/remote/connector.go +++ b/pkg/mcs/remote/connector.go @@ -472,7 +472,7 @@ func (c *Connector) deleteServiceImport(export *mcsevent.ServiceExportEvent) err func (c *Connector) rejectServiceExport(svcExportEvt *mcsevent.ServiceExportEvent) error { ctx := c.context.(*conn.ConnectorContext) export := svcExportEvt.ServiceExport - //reason := svcExportEvt.Data["reason"] + //reason := svcExportEvt.data["reason"] reason := svcExportEvt.Error if ctx.ClusterKey == svcExportEvt.ClusterKey() { diff --git a/pkg/sidecar/providers/pipy/driver/debugger_sidecar.go b/pkg/sidecar/providers/pipy/driver/debugger_sidecar.go index d1836d4ad..bfed764a9 100644 --- a/pkg/sidecar/providers/pipy/driver/debugger_sidecar.go +++ b/pkg/sidecar/providers/pipy/driver/debugger_sidecar.go @@ -17,10 +17,9 @@ func (sd PipySidecarDriver) getSidecarConfig(pod *v1.Pod, url string) string { minPort := 16000 maxPort := 18000 - // #nosec G404 portFwdRequest := debugger.PortForward{ Pod: pod, - LocalPort: rand.Intn(maxPort-minPort) + minPort, + LocalPort: rand.Intn(maxPort-minPort) + minPort, // #nosec G404 PodPort: 15000, Stop: make(chan struct{}), Ready: make(chan struct{}), diff --git a/pkg/utils/mtls.go b/pkg/utils/mtls.go index 5f6029652..b7844834a 100644 --- a/pkg/utils/mtls.go +++ b/pkg/utils/mtls.go @@ -28,9 +28,8 @@ func setupMutualTLS(insecure bool, serverName string, certPem []byte, keyPem []b return nil, fmt.Errorf("[grpc][mTLS][%s] Failed to append client certs", serverName) } - // #nosec G402 tlsConfig := tls.Config{ - InsecureSkipVerify: insecure, + InsecureSkipVerify: insecure, // #nosec G402 ServerName: serverName, ClientAuth: tls.RequireAndVerifyClientCert, Certificates: []tls.Certificate{certif}, diff --git a/pkg/utils/repo.go b/pkg/utils/repo.go index 54095d350..6b6f0440c 100644 --- a/pkg/utils/repo.go +++ b/pkg/utils/repo.go @@ -73,9 +73,9 @@ func GetDefaultIngressPath() string { } // GatewayCodebasePath get the codebase URL for the gateway in specified namespace -// inherit hierarchy: /base/gateways -> /local/gateways -> /local/gw/[ns] -func GatewayCodebasePath(namespace string) string { - return fmt.Sprintf("/local/gw/%s", namespace) +// inherit hierarchy: /base/gateways -> /local/gateways -> /local/gw/[ns]/[name] +func GatewayCodebasePath(namespace, name string) string { + return fmt.Sprintf("/local/gw/%s/%s", namespace, name) } // GetDefaultGatewaysPath returns the path to the gateways codebase. diff --git a/pkg/utils/util.go b/pkg/utils/util.go index d006e1661..0744bc76e 100644 --- a/pkg/utils/util.go +++ b/pkg/utils/util.go @@ -19,7 +19,7 @@ func SimpleHash(obj interface{}) string { hash, err := hashstructure.Hash(obj, hashstructure.FormatV2, nil) if err != nil { - log.Error().Msgf("Not able convert Data to hash, error: %s", err.Error()) + log.Error().Msgf("Not able convert data to hash, error: %s", err.Error()) return "" } diff --git a/pkg/webhook/gateway/gateway_webhook.go b/pkg/webhook/gateway/gateway_webhook.go index 3504ad7d2..321656261 100644 --- a/pkg/webhook/gateway/gateway_webhook.go +++ b/pkg/webhook/gateway/gateway_webhook.go @@ -104,7 +104,7 @@ func (r *register) GetWebhooks() ([]admissionregv1.MutatingWebhook, []admissionr func (r *register) GetHandlers() map[string]http.Handler { return map[string]http.Handler{ constants.GatewayMutatingWebhookPath: webhook.DefaultingWebhookFor(r.Scheme, newDefaulter(r.KubeClient, r.gatewayAPIClient, r.Configurator, r.MeshName, r.FSMVersion)), - constants.GatewayValidatingWebhookPath: webhook.ValidatingWebhookFor(r.Scheme, newValidator(r.KubeClient, r.Configurator)), + constants.GatewayValidatingWebhookPath: webhook.ValidatingWebhookFor(r.Scheme, newValidator(r.KubeClient, r.gatewayAPIClient, r.Configurator)), } } @@ -139,7 +139,7 @@ func (w *defaulter) SetDefaults(obj interface{}) { } log.Debug().Msgf("Default Webhook, name=%s", gateway.Name) - log.Debug().Msgf("Before setting default values, spec=%v", gateway.Spec) + log.Debug().Msgf("Before setting default values: %v", gateway) gatewayClass, err := w.gatewayAPIClient. GatewayV1(). @@ -164,12 +164,13 @@ func (w *defaulter) SetDefaults(obj interface{}) { gateway.Labels[constants.FSMAppVersionLabelKey] = w.fsmVersion gateway.Labels[constants.AppLabel] = constants.FSMGatewayName - log.Debug().Msgf("After setting default values, spec=%v", gateway.Spec) + log.Debug().Msgf("After setting default values: %v", gateway) } type validator struct { - kubeClient kubernetes.Interface - cfg configurator.Configurator + kubeClient kubernetes.Interface + gatewayAPIClient gatewayApiClientset.Interface + cfg configurator.Configurator } // RuntimeObject returns the runtime object of gateway @@ -192,10 +193,11 @@ func (w *validator) ValidateDelete(_ interface{}) error { return nil } -func newValidator(kubeClient kubernetes.Interface, cfg configurator.Configurator) *validator { +func newValidator(kubeClient kubernetes.Interface, gatewayAPIClient gatewayApiClientset.Interface, cfg configurator.Configurator) *validator { return &validator{ - kubeClient: kubeClient, - cfg: cfg, + kubeClient: kubeClient, + gatewayAPIClient: gatewayAPIClient, + cfg: cfg, } } @@ -205,6 +207,20 @@ func (w *validator) doValidation(obj interface{}) error { return nil } + gatewayClass, err := w.gatewayAPIClient. + GatewayV1(). + GatewayClasses(). + Get(context.TODO(), string(gateway.Spec.GatewayClassName), metav1.GetOptions{}) + if err != nil { + log.Error().Msgf("failed to get gatewayclass %s", gateway.Spec.GatewayClassName) + return nil + } + + if gatewayClass.Spec.ControllerName != constants.GatewayController { + log.Warn().Msgf("class controller of Gateway %s/%s is not %s", gateway.Namespace, gateway.Name, constants.GatewayController) + return nil + } + var errorList field.ErrorList if !version.IsCELValidationEnabled(w.kubeClient) { errorList = append(errorList, gwv1validation.ValidateGateway(gateway)...) @@ -265,7 +281,7 @@ func (w *validator) validateSecretsExistence(gateway *gwv1.Gateway, c gwv1.Liste for j, ref := range c.TLS.CertificateRefs { if string(*ref.Kind) == "Secret" && string(*ref.Group) == "" { - ns := gwutils.Namespace(ref.Namespace, gateway.Namespace) + ns := gwutils.NamespaceDerefOr(ref.Namespace, gateway.Namespace) name := string(ref.Name) path := field.NewPath("spec"). diff --git a/scripts/k3d-with-registry.sh b/scripts/k3d-with-registry.sh index e8dd295a8..439c5b753 100755 --- a/scripts/k3d-with-registry.sh +++ b/scripts/k3d-with-registry.sh @@ -11,7 +11,7 @@ K3D_NAMESPACED_INGRESS_ENABLE="${K3D_NAMESPACED_INGRESS_ENABLE:-false}" K3D_GATEWAY_API_ENABLE="${K3D_GATEWAY_API_ENABLE:-false}" K3D_FLB_ENABLE="${K3D_FLB_ENABLE:-false}" K3D_SERVICELB_ENABLE="${K3D_SERVICELB_ENABLE:-false}" -K3D_IMAGE="${K3D_IMAGE:-rancher/k3s:v1.21.11-k3s1}" +K3D_IMAGE="${K3D_IMAGE:-rancher/k3s:v1.25.16-k3s4}" # shellcheck disable=SC2086 jq_cluster_exists=".[] | select(.name == \"$K3D_CLUSTER_NAME\")" diff --git a/tests/e2e/e2e_gatewayapi_test.go b/tests/e2e/e2e_gatewayapi_test.go index a49379162..0f26c90e2 100644 --- a/tests/e2e/e2e_gatewayapi_test.go +++ b/tests/e2e/e2e_gatewayapi_test.go @@ -52,7 +52,12 @@ var _ = FSMDescribe("Test traffic among FSM Gateway", installOpts.EnableServiceLB = true Expect(Td.InstallFSM(installOpts)).To(Succeed()) - Expect(Td.WaitForPodsRunningReady(Td.FsmNamespace, 3, nil)).To(Succeed()) + Expect(Td.WaitForPodsRunningReady(Td.FsmNamespace, 3, &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app.kubernetes.io/instance": "fsm", + "app.kubernetes.io/name": "flomesh.io", + }, + })).To(Succeed()) testDeployFSMGateway() @@ -151,7 +156,7 @@ func testDeployFSMGateway() { Data: map[string]string{ "values.yaml": ` fsm: - fsmGateway: + gateway: replicas: 2 resources: requests: