From 8617e7589f250d3973854a7f48c20c8e5eaefa35 Mon Sep 17 00:00:00 2001 From: Ivan Matmati Date: Thu, 11 May 2023 10:31:57 +0200 Subject: [PATCH] MINOR: check GW API crd version --- go.mod | 1 + go.sum | 1 + main.go | 7 ++-- pkg/controller/controller.go | 4 +++ pkg/gateways/gateways.go | 29 +++++++++++------ pkg/k8s/main.go | 63 ++++++++++++++++++++++++++++++++---- 6 files changed, 86 insertions(+), 19 deletions(-) diff --git a/go.mod b/go.mod index a0b47f6d..57fa96f0 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/valyala/fasthttp v1.47.0 go.uber.org/automaxprocs v1.5.2 k8s.io/api v0.27.2 + k8s.io/apiextensions-apiserver v0.27.2 k8s.io/apimachinery v0.27.2 k8s.io/client-go v0.27.2 sigs.k8s.io/controller-runtime v0.15.0 diff --git a/go.sum b/go.sum index 519b0117..c3037919 100644 --- a/go.sum +++ b/go.sum @@ -542,6 +542,7 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh k8s.io/api v0.27.2 h1:+H17AJpUMvl+clT+BPnKf0E3ksMAzoBBg7CntpSuADo= k8s.io/api v0.27.2/go.mod h1:ENmbocXfBT2ADujUXcBhHV55RIT31IIEvkntP6vZKS4= k8s.io/apiextensions-apiserver v0.27.2 h1:iwhyoeS4xj9Y7v8YExhUwbVuBhMr3Q4bd/laClBV6Bo= +k8s.io/apiextensions-apiserver v0.27.2/go.mod h1:Oz9UdvGguL3ULgRdY9QMUzL2RZImotgxvGjdWRq6ZXQ= k8s.io/apimachinery v0.27.2 h1:vBjGaKKieaIreI+oQwELalVG4d8f3YAMNpWLzDXkxeg= k8s.io/apimachinery v0.27.2/go.mod h1:XNfZ6xklnMCOGGFNqXG7bUrQCoR04dh/E7FprV6pb+E= k8s.io/client-go v0.27.2 h1:vDLSeuYvCHKeoQRhCXjxXO45nHVv2Ip4Fe0MfioMrhE= diff --git a/main.go b/main.go index c7f1dd10..c1b14c6e 100644 --- a/main.go +++ b/main.go @@ -119,9 +119,12 @@ func main() { WithRestClientSet(k.GetRestClientset()). WithArgs(osArgs).Build() - go k.MonitorChanges(eventChan, stop, osArgs) - go c.Start() + isGatewayAPIInstalled := k.IsGatewayAPIInstalled(osArgs.GatewayControllerName) + + c.SetGatewayAPIInstalled(isGatewayAPIInstalled) + go k.MonitorChanges(eventChan, stop, osArgs, isGatewayAPIInstalled) + go c.Start() // Catch QUIT signals signalC := make(chan os.Signal, 1) signal.Notify(signalC, os.Interrupt, syscall.SIGTERM, syscall.SIGUSR1) diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index c8a1dc53..94ca8e52 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -298,3 +298,7 @@ func (c *HAProxyController) clean(failedSync bool) { c.reload = false c.restart = false } + +func (c *HAProxyController) SetGatewayAPIInstalled(gatewayAPIInstalled bool) { + c.gatewayManager.SetGatewayAPIInstalled(gatewayAPIInstalled) +} diff --git a/pkg/gateways/gateways.go b/pkg/gateways/gateways.go index 37196548..b6f10160 100644 --- a/pkg/gateways/gateways.go +++ b/pkg/gateways/gateways.go @@ -47,6 +47,7 @@ const ( //nolint:golint type GatewayManager interface { ManageGateway() bool + SetGatewayAPIInstalled(bool) } func New(k8sStore store.K8s, @@ -54,7 +55,7 @@ func New(k8sStore store.K8s, osArgs utils.OSArgs, k8sRestClient client.Client, ) GatewayManager { - return GatewayManagerImpl{ + return &GatewayManagerImpl{ k8sStore: k8sStore, haproxyClient: haproxyClient, osArgs: osArgs, @@ -69,18 +70,22 @@ func New(k8sStore store.K8s, //nolint:golint type GatewayManagerImpl struct { - haproxyClient api.HAProxyClient - statusManager StatusManager - frontends map[string]struct{} - gateways map[string]struct{} - listenersByRoute map[string][]store.Listener - backends map[string]struct{} - serversByBackend map[string][]string - k8sStore store.K8s - osArgs utils.OSArgs + haproxyClient api.HAProxyClient + statusManager StatusManager + frontends map[string]struct{} + gateways map[string]struct{} + listenersByRoute map[string][]store.Listener + backends map[string]struct{} + serversByBackend map[string][]string + k8sStore store.K8s + osArgs utils.OSArgs + gatewayAPIInstalled bool } func (gm GatewayManagerImpl) ManageGateway() bool { + if gm.osArgs.GatewayControllerName == "" || !gm.gatewayAPIInstalled { + return false + } gm.clean() gm.manageGatewayClass() @@ -669,3 +674,7 @@ func (gm *GatewayManagerImpl) resetStatuses() { } } } + +func (gm *GatewayManagerImpl) SetGatewayAPIInstalled(gatewayAPIInstalled bool) { + gm.gatewayAPIInstalled = gatewayAPIInstalled +} diff --git a/pkg/k8s/main.go b/pkg/k8s/main.go index 342cf8c3..b9c6fac7 100644 --- a/pkg/k8s/main.go +++ b/pkg/k8s/main.go @@ -15,6 +15,7 @@ package k8s import ( + "context" "errors" "os" "path/filepath" @@ -31,6 +32,9 @@ import ( crinformers "github.com/haproxytech/kubernetes-ingress/crs/generated/informers/externalversions" "github.com/haproxytech/kubernetes-ingress/pkg/ingress" "github.com/haproxytech/kubernetes-ingress/pkg/utils" + crdclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + errGw "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" gatewayclientset "sigs.k8s.io/gateway-api/pkg/client/clientset/versioned" scheme "sigs.k8s.io/gateway-api/pkg/client/clientset/versioned/scheme" @@ -44,6 +48,7 @@ const ( TRACE_API = false //nolint:golint,stylecheck CRSGroupVersionV1alpha1 = "core.haproxy.org/v1alpha1" CRSGroupVersionV1alpha2 = "core.haproxy.org/v1alpha2" + GATEWAY_API_VERSION = "v0.5.1" //nolint:golint,stylecheck ) var ErrIgnored = errors.New("ignored resource") @@ -51,8 +56,9 @@ var ErrIgnored = errors.New("ignored resource") type K8s interface { GetRestClientset() client.Client GetClientset() *k8sclientset.Clientset - MonitorChanges(eventChan chan SyncDataEvent, stop chan struct{}, osArgs utils.OSArgs) + MonitorChanges(eventChan chan SyncDataEvent, stop chan struct{}, osArgs utils.OSArgs, gatewayAPIInstalled bool) UpdatePublishService(ingresses []*ingress.Ingress, publishServiceAddresses []string) + IsGatewayAPIInstalled(gatewayControllerName string) bool } // A Custom Resource interface @@ -77,6 +83,8 @@ type k8s struct { syncPeriod time.Duration cacheResyncPeriod time.Duration disableSvcExternalName bool // CVE-2021-25740 + crdClient *crdclientset.Clientset + gatewayAPIInstalled bool } func New(osArgs utils.OSArgs, whitelist map[string]struct{}, publishSvc *utils.NamespaceValue) K8s { //nolint:ireturn @@ -104,6 +112,11 @@ func New(osArgs utils.OSArgs, whitelist map[string]struct{}, publishSvc *utils.N logger.Print("Gateway API not present") } + crdClient, err := crdclientset.NewForConfig(restconfig) + if err != nil { + logger.Error("CRD API client not present") + } + prefix, _ := utils.GetPodPrefix(os.Getenv("POD_NAME")) k := k8s{ builtInClient: builtInClient, @@ -118,6 +131,7 @@ func New(osArgs utils.OSArgs, whitelist map[string]struct{}, publishSvc *utils.N disableSvcExternalName: osArgs.DisableServiceExternalName, gatewayClient: gatewayClient, gatewayRestClient: gatewayRestClient, + crdClient: crdClient, } // alpha1 is deprecated k.registerCoreCR(NewGlobalCRV1Alpha1(), CRSGroupVersionV1alpha1) @@ -145,14 +159,13 @@ func (k k8s) UpdatePublishService(ingresses []*ingress.Ingress, publishServiceAd } } -func (k k8s) MonitorChanges(eventChan chan SyncDataEvent, stop chan struct{}, osArgs utils.OSArgs) { +func (k k8s) MonitorChanges(eventChan chan SyncDataEvent, stop chan struct{}, osArgs utils.OSArgs, gatewayAPIInstalled bool) { informersSynced := &[]cache.InformerSynced{} - needGatewayAPIInformers := k.isGatewayAPIInstalled() && osArgs.GatewayControllerName != "" k.runPodInformer(eventChan, stop, informersSynced) for _, namespace := range k.whiteListedNS { k.runInformers(eventChan, stop, namespace, informersSynced) k.runCRInformers(eventChan, stop, namespace, informersSynced) - if needGatewayAPIInformers { + if gatewayAPIInstalled { k.runInformersGwAPI(eventChan, stop, namespace, informersSynced) } } @@ -320,7 +333,43 @@ func getWhitelistedNS(whitelist map[string]struct{}, cfgMapNS string) []string { return namespaces } -func (k k8s) isGatewayAPIInstalled() bool { - _, err := k.crClient.DiscoveryClient.ServerResourcesForGroupVersion("gateway.networking.k8s.io/v1beta1") - return err == nil +func (k k8s) IsGatewayAPIInstalled(gatewayControllerName string) (installed bool) { + installed = true + defer func() { + k.gatewayAPIInstalled = installed + }() + gatewayCrd, err := k.crdClient.ApiextensionsV1().CustomResourceDefinitions().Get(context.Background(), "gateways.gateway.networking.k8s.io", metav1.GetOptions{}) + if err != nil { + var errStatus *errGw.StatusError + if !errors.As(err, &errStatus) || errStatus.ErrStatus.Code != 404 { + logger.Error(err) + return false + } + } + + if gatewayCrd.Name == "" { + if gatewayControllerName != "" { + logger.Errorf("No gateway api is installed, please install experimental yaml version %s", GATEWAY_API_VERSION) + } + return false + } + + log := logger.Warningf + if gatewayControllerName != "" { + log = logger.Errorf + } + + version := gatewayCrd.Annotations["gateway.networking.k8s.io/bundle-version"] + if version != GATEWAY_API_VERSION { + log("Unsupported version '%s' of gateway api is installed, please install experimental yaml version %s", version, GATEWAY_API_VERSION) + installed = false + } + + // gatewayCrd is not nil so gateway API is present + tcprouteCrd, err := k.crdClient.ApiextensionsV1().CustomResourceDefinitions().Get(context.Background(), "tcproutes.gateway.networking.k8s.io", metav1.GetOptions{}) + if tcprouteCrd == nil || err != nil { + log("No tcproute crd is installed, please install experimental yaml version %s", GATEWAY_API_VERSION) + installed = false + } + return }