more cleanup - separate control plane (#10413)

- bring in control plane to gateway
- don't start gloo any-more.
- use upstream go control plane
- various other more minor code clean-ups.

projects/gateway/pkg/validation/validator.go

@@ -16,7 +16,8 @@ import (
 	sologatewayv1 "github.com/solo-io/gloo/projects/gateway/pkg/api/v1"
 	"github.com/solo-io/gloo/projects/gateway/pkg/translator"
 	"github.com/solo-io/gloo/projects/gateway/pkg/utils"
-	k8sgwvalidation "github.com/solo-io/gloo/projects/gateway2/validation"
+
+	//	k8sgwvalidation "github.com/solo-io/gloo/projects/gateway2/validation"
 	"github.com/solo-io/gloo/projects/gloo/pkg/api/grpc/validation"
 	gloov1 "github.com/solo-io/gloo/projects/gloo/pkg/api/v1"
 	syncerValidation "github.com/solo-io/gloo/projects/gloo/pkg/syncer/validation"
@@ -288,8 +289,8 @@ func (v *validator) validateProxiesAndExtensions(ctx context.Context, snapshot *
 
 	validatingK8sGateway := isK8sGatewayProxy(opts.Resource)
 	if validatingK8sGateway {
-		proxies, errs = k8sgwvalidation.TranslateK8sGatewayProxies(ctx, snapshot, opts.Resource)
-	} else {
+		//		proxies, errs = k8sgwvalidation.TranslateK8sGatewayProxies(ctx, snapshot, opts.Resource)
+		//	} else {
 		proxies, errs = v.translateGlooEdgeProxies(ctx, snapshot, opts.collectAllErrors)
 	}
 
@@ -326,9 +327,9 @@ func (v *validator) validateProxiesAndExtensions(ctx context.Context, snapshot *
 		// error aggregation below. Note that we do not care about allowWarnings, as they are not
 		// used in this case, as we do not do a full translation and use a 'dummy' proxy
 		if validatingK8sGateway {
-			if err = k8sgwvalidation.GetSimpleErrorFromGlooValidation(glooReports, proxy); err != nil {
-				errs = multierror.Append(errs, err)
-			}
+			//	if err = k8sgwvalidation.GetSimpleErrorFromGlooValidation(glooReports, proxy); err != nil {
+			//		errs = multierror.Append(errs, err)
+			//	}
 			continue
 		}
 

projects/gateway2/controller/scheme.go

@@ -0,0 +1,62 @@
+package controller
+
+import (
+	"fmt"
+
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	discoveryv1 "k8s.io/api/discovery/v1"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	gwv1 "sigs.k8s.io/gateway-api/apis/v1"
+	gwv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
+	gwv1b1 "sigs.k8s.io/gateway-api/apis/v1beta1"
+
+	sologatewayv1alpha1 "github.com/solo-io/gloo/projects/gateway2/api/v1alpha1"
+)
+
+// SchemeBuilder contains all the Schemes for registering the CRDs with which Gloo Gateway interacts.
+// We share one SchemeBuilder as there's no harm in registering all I/O types internally.
+var SchemeBuilder = runtime.SchemeBuilder{
+	// K8s Gateway API resources
+	gwv1.Install,
+	gwv1a2.Install,
+	gwv1b1.Install,
+
+	// Kubernetes Core resources
+	corev1.AddToScheme,
+	appsv1.AddToScheme,
+	discoveryv1.AddToScheme,
+
+	// Register the apiextensions API group
+	apiextensionsv1.AddToScheme,
+
+	// Solo Kubernetes Gateway API resources
+	sologatewayv1alpha1.Install,
+}
+
+func AddToScheme(s *runtime.Scheme) error {
+	return SchemeBuilder.AddToScheme(s)
+}
+
+// DefaultScheme returns a scheme with all the types registered for Gloo Gateway
+// We intentionally do not perform this operation in an init!!
+// See https://github.com/solo-io/gloo/pull/9692 for context
+func DefaultScheme() *runtime.Scheme {
+	s := runtime.NewScheme()
+	_ = AddToScheme(s)
+	return s
+}
+
+// GatewayScheme unconditionally includes the default and required Gateway API schemes.
+// Use the Default scheme with AddGatewayV1A2Scheme to conditionally add the v1alpha2 scheme.
+func GatewayScheme() *runtime.Scheme {
+	s := DefaultScheme()
+	if err := gwv1a2.Install(s); err != nil {
+		panic(fmt.Sprintf("Failed to install gateway v1alpha2 scheme: %v", err))
+	}
+	if err := gwv1b1.Install(s); err != nil {
+		panic(fmt.Sprintf("Failed to install gateway v1beta1 scheme: %v", err))
+	}
+	return s
+}

projects/gateway2/controller/start.go

@@ -10,14 +10,14 @@ import (
 	glooschemes "github.com/solo-io/gloo/pkg/schemes"
 	"github.com/solo-io/go-utils/contextutils"
 
-	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/sets"
 	ctrl "sigs.k8s.io/controller-runtime"
 
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	czap "sigs.k8s.io/controller-runtime/pkg/log/zap"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 
+	envoycache "github.com/envoyproxy/go-control-plane/pkg/cache/v3"
 	"github.com/solo-io/gloo/projects/gateway2/deployer"
 	"github.com/solo-io/gloo/projects/gateway2/extensions2"
 	"github.com/solo-io/gloo/projects/gateway2/extensions2/common"
@@ -30,17 +30,11 @@ import (
 	"github.com/solo-io/gloo/projects/gateway2/utils/krtutil"
 	"github.com/solo-io/gloo/projects/gateway2/wellknown"
 	glookubev1 "github.com/solo-io/gloo/projects/gloo/pkg/api/v1/kube/apis/gloo.solo.io/v1"
-	"github.com/solo-io/gloo/projects/gloo/pkg/bootstrap"
-	"github.com/solo-io/gloo/projects/gloo/pkg/syncer"
-	"github.com/solo-io/solo-kit/pkg/api/v2/reporter"
 	uzap "go.uber.org/zap"
 	istiokube "istio.io/istio/pkg/kube"
-	"istio.io/istio/pkg/kube/kclient"
 	"istio.io/istio/pkg/kube/krt"
 	istiolog "istio.io/istio/pkg/log"
-	corev1 "k8s.io/api/core/v1"
 	apiv1 "sigs.k8s.io/gateway-api/apis/v1"
-	gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1"
 )
 
 const (
@@ -49,30 +43,26 @@ const (
 	AutoProvision = true
 )
 
+type SetupOpts struct {
+	Cache               envoycache.SnapshotCache
+	ExtraGatewayClasses []string
+
+	KrtDebugger *krt.DebugHandler
+
+	XdsHost string
+	XdsPort int32
+}
+
 var setupLog = ctrl.Log.WithName("setup")
 
 type StartConfig struct {
 	Dev        bool
-	SetupOpts  *bootstrap.SetupOpts
+	SetupOpts  *SetupOpts
 	RestConfig *rest.Config
 	// ExtensionsFactory is the factory function which will return an extensions.K8sGatewayExtensions
 	// This is responsible for producing the extension points that this controller requires
 	ExtraPlugins []extensionsplug.Plugin
 
-	// GlooStatusReporter is the shared reporter from setup_syncer that reports as 'gloo',
-	// it is used to report on Upstreams and Proxies after xds translation.
-	// this is required because various upstream tests expect a certain reporter for Upstreams
-	// TODO: remove the other reporter and only use this one, no need for 2 different reporters
-	GlooStatusReporter reporter.StatusReporter
-
-	// KubeGwStatusReporter is used within any StatusPlugins that must persist a GE-classic style status
-	// TODO: as mentioned above, this should be removed: https://github.com/solo-io/solo-projects/issues/7055
-	KubeGwStatusReporter reporter.StatusReporter
-
-	// SyncerExtensions is a list of extensions, the kube gw controller will use these to get extension-specific
-	// errors & warnings for any Proxies it generates
-	SyncerExtensions []syncer.TranslatorSyncerExtension
-
 	Client istiokube.Client
 
 	AugmentedPods krt.Collection[krtcollections.LocalityPod]
@@ -106,7 +96,7 @@ func NewControllerBuilder(ctx context.Context, cfg StartConfig) (*ControllerBuil
 	ctrl.SetLogger(czap.New(opts...))
 	istiolog.Configure(loggingOptions)
 
-	scheme := glooschemes.DefaultScheme()
+	scheme := DefaultScheme()
 
 	// Extend the scheme if the TCPRoute CRD exists.
 	if err := glooschemes.AddGatewayV1A2Scheme(cfg.RestConfig, scheme); err != nil {
@@ -140,41 +130,17 @@ func NewControllerBuilder(ctx context.Context, cfg StartConfig) (*ControllerBuil
 	mgr.AddReadyzCheck("ready-ping", healthz.Ping)
 
 	setupLog.Info("initializing k8sgateway extensions")
-	secretClient := kclient.New[*corev1.Secret](cfg.Client)
-	k8sSecretsRaw := krt.WrapClient(secretClient, krt.WithStop(ctx.Done()), krt.WithName("Secrets") /* no debug here - we don't want raw secrets printed*/)
-	k8sSecrets := krt.NewCollection(k8sSecretsRaw, func(kctx krt.HandlerContext, i *corev1.Secret) *ir.Secret {
-		res := ir.Secret{
-			ObjectSource: ir.ObjectSource{
-				Group:     "",
-				Kind:      "Secret",
-				Namespace: i.Namespace,
-				Name:      i.Name,
-			},
-			Obj:  i,
-			Data: i.Data,
-		}
-		return &res
-	}, cfg.KrtOptions.ToOptions("secrets")...)
-	secrets := map[schema.GroupKind]krt.Collection[ir.Secret]{
-		{Group: "", Kind: "Secret"}: k8sSecrets,
-	}
-
-	refgrantsCol := krt.WrapClient(kclient.New[*gwv1beta1.ReferenceGrant](cfg.Client), cfg.KrtOptions.ToOptions("RefGrants")...)
-	refgrants := krtcollections.NewRefGrantIndex(refgrantsCol)
 	cli, err := versioned.NewForConfig(cfg.RestConfig)
 	if err != nil {
 		return nil, err
 	}
-	commoncol := common.CommonCollections{
-		OurClient: cli,
-		Client:    cfg.Client,
-		KrtOpts:   cfg.KrtOptions,
-		Secrets:   krtcollections.NewSecretIndex(secrets, refgrants),
-		Pods:      cfg.AugmentedPods,
-		Settings:  cfg.Settings,
-		RefGrants: refgrants,
-	}
-
+	commoncol := common.NewCommonCollections(
+		cfg.KrtOptions,
+		cfg.Client,
+		cli,
+		cfg.InitialSettings,
+		cfg.Settings,
+	)
 	gwClasses := sets.New(append(cfg.SetupOpts.ExtraGatewayClasses, wellknown.GatewayClassName)...)
 	isOurGw := func(gw *apiv1.Gateway) bool {
 		return gwClasses.Has(string(gw.Spec.GatewayClassName))
@@ -188,7 +154,6 @@ func NewControllerBuilder(ctx context.Context, cfg StartConfig) (*ControllerBuil
 		wellknown.GatewayControllerName,
 		mgr,
 		cfg.Client,
-		cfg.AugmentedPods,
 		cfg.UniqueClients,
 		pluginFactoryWithBuiltin(cfg.ExtraPlugins),
 		commoncol,
@@ -223,7 +188,7 @@ func (c *ControllerBuilder) Start(ctx context.Context) error {
 	logger.Info("starting gateway controller")
 	// GetXdsAddress waits for gloo-edge to populate the xds address of the server.
 	// in the future this logic may move here and be duplicated.
-	xdsHost, xdsPort := c.cfg.SetupOpts.GetXdsAddress(ctx)
+	xdsHost, xdsPort := c.cfg.SetupOpts.XdsHost, c.cfg.SetupOpts.XdsPort
 	if xdsHost == "" {
 		return ctx.Err()
 	}

projects/gateway2/deployer/deployer_test.go

@@ -7,6 +7,7 @@ import (
 
 	envoy_config_bootstrap "github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v3"
 	_ "github.com/envoyproxy/go-control-plane/envoy/extensions/upstreams/http/v3"
+	"github.com/ghodss/yaml"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"github.com/onsi/gomega/types"
@@ -19,15 +20,13 @@ import (
 	glooutils "github.com/solo-io/gloo/projects/gloo/pkg/utils"
 	"github.com/solo-io/gloo/projects/gloo/pkg/xds"
 	"github.com/solo-io/gloo/test/gomega/matchers"
-	"github.com/solo-io/solo-kit/pkg/api/v1/resources/core"
-	"github.com/solo-io/solo-kit/pkg/utils/protoutils"
+	jsonpb "google.golang.org/protobuf/encoding/protojson"
+	"google.golang.org/protobuf/proto"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
-	"k8s.io/apimachinery/pkg/util/yaml"
-	"k8s.io/utils/pointer"
 	"k8s.io/utils/ptr"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
@@ -46,18 +45,16 @@ import (
 // testBootstrap implements resources.Resource in order to use protoutils.UnmarshalYAML
 // this is hacky but it seems more stable/concise than map-casting all the way down
 // to the field we need.
-type testBootstrap struct {
-	envoy_config_bootstrap.Bootstrap
-}
 
-func (t *testBootstrap) SetMetadata(meta *core.Metadata) {}
+func unmarshalYaml(data []byte, into proto.Message) error {
+	jsn, err := yaml.YAMLToJSON(data)
+	if err != nil {
+		return err
+	}
 
-func (t *testBootstrap) Equal(_ any) bool {
-	return false
-}
+	var j jsonpb.UnmarshalOptions
 
-func (t *testBootstrap) GetMetadata() *core.Metadata {
-	return nil
+	return j.Unmarshal(jsn, into)
 }
 
 type clientObjects []client.Object
@@ -106,10 +103,10 @@ func (objs *clientObjects) findConfigMap(namespace, name string) *corev1.ConfigM
 	return nil
 }
 
-func (objs *clientObjects) getEnvoyConfig(namespace, name string) *testBootstrap {
+func (objs *clientObjects) getEnvoyConfig(namespace, name string) *envoy_config_bootstrap.Bootstrap {
 	cm := objs.findConfigMap(namespace, name).Data
-	var bootstrapCfg testBootstrap
-	err := protoutils.UnmarshalYAML([]byte(cm["envoy.yaml"]), &bootstrapCfg)
+	var bootstrapCfg envoy_config_bootstrap.Bootstrap
+	err := unmarshalYaml([]byte(cm["envoy.yaml"]), &bootstrapCfg)
 	ExpectWithOffset(1, err).ToNot(HaveOccurred())
 	return &bootstrapCfg
 }
@@ -767,10 +764,10 @@ var _ = Describe("Deployer", func() {
 				params := fullyDefinedGatewayParameters(wellknown.DefaultGatewayParametersName, defaultNamespace)
 				params.Spec.Kube.PodTemplate.LivenessProbe = generateLivenessProbe()
 				params.Spec.Kube.PodTemplate.ReadinessProbe = generateReadinessProbe()
-				params.Spec.Kube.PodTemplate.TerminationGracePeriodSeconds = pointer.Int(5)
+				params.Spec.Kube.PodTemplate.TerminationGracePeriodSeconds = ptr.To(5)
 				params.Spec.Kube.PodTemplate.GracefulShutdown = &gw2_v1alpha1.GracefulShutdownSpec{
-					Enabled:          pointer.Bool(true),
-					SleepTimeSeconds: pointer.Int(7),
+					Enabled:          ptr.To(true),
+					SleepTimeSeconds: ptr.To(7),
 				}
 				return params
 			}