From 05dd7533d848cf838bf71ac8f021f9310ab6cd0a Mon Sep 17 00:00:00 2001 From: Yuval Kohavi Date: Fri, 8 Nov 2024 10:35:54 -0500 Subject: [PATCH 1/5] move snapshots interface to localhost --- projects/gloo/pkg/servers/admin/server.go | 6 ++ projects/gloo/pkg/syncer/setup/start_func.go | 88 +++++++++++++++++++- 2 files changed, 93 insertions(+), 1 deletion(-) diff --git a/projects/gloo/pkg/servers/admin/server.go b/projects/gloo/pkg/servers/admin/server.go index 5c3edb1deab..eb02f063489 100644 --- a/projects/gloo/pkg/servers/admin/server.go +++ b/projects/gloo/pkg/servers/admin/server.go @@ -20,6 +20,7 @@ func ServerHandlers(ctx context.Context, history iosnapshot.History, dbg *krt.De response := history.GetInputSnapshot(ctx) respondJson(w, response) }) + profiles["/snapshots/input"] = "Input Snapshot" // The Edge Snapshot is intended to return a representation of the ApiSnapshot object that the Control Plane // manages internally. This is not intended to be consumed by users, but instead be a mechanism to feed this @@ -28,6 +29,7 @@ func ServerHandlers(ctx context.Context, history iosnapshot.History, dbg *krt.De response := history.GetEdgeApiSnapshot(ctx) respondJson(w, response) }) + profiles["/snapshots/edge"] = "Edge Snapshot" // The Proxy Snapshot is intended to return a representation of the Proxies within the ApiSnapshot object. // Proxies may either be persisted in etcD or in-memory, so this Api provides a single mechansim to access @@ -36,6 +38,7 @@ func ServerHandlers(ctx context.Context, history iosnapshot.History, dbg *krt.De response := history.GetProxySnapshot(ctx) respondJson(w, response) }) + profiles["/snapshots/proxies"] = "Proxy Snapshot" // The xDS Snapshot is intended to return the full in-memory xDS cache that the Control Plane manages // and serves up to running proxies. @@ -43,9 +46,12 @@ func ServerHandlers(ctx context.Context, history iosnapshot.History, dbg *krt.De response := history.GetXdsSnapshot(ctx) respondJson(w, response) }) + profiles["/snapshots/xds"] = "XDS Snapshot" + m.HandleFunc("/snapshots/krt", func(w http.ResponseWriter, r *http.Request) { writeJSON(w, dbg, r) }) + profiles["/snapshots/krt"] = "KRT Snapshot" } } diff --git a/projects/gloo/pkg/syncer/setup/start_func.go b/projects/gloo/pkg/syncer/setup/start_func.go index 478fcf729e8..92ebd47439e 100644 --- a/projects/gloo/pkg/syncer/setup/start_func.go +++ b/projects/gloo/pkg/syncer/setup/start_func.go @@ -1,7 +1,12 @@ package setup import ( + "bytes" "context" + "fmt" + "net/http" + "os" + "sort" "github.com/solo-io/gloo/projects/gloo/pkg/servers/admin" "github.com/solo-io/gloo/projects/gloo/pkg/servers/iosnapshot" @@ -15,6 +20,10 @@ import ( "github.com/solo-io/gloo/projects/gloo/pkg/bootstrap" ) +const ( + AdminPort = 9095 +) + // StartFunc represents a function that will be called with the initialized bootstrap.Opts // and Extensions. This is invoked each time the setup_syncer is executed // (which runs whenever the Setting CR is modified) @@ -63,8 +72,85 @@ func AdminServerStartFunc(history iosnapshot.History, dbg *krt.DebugHandler) Sta // NOTE: There is a slight difference in how we run this server -vs- how we used to run it // In the past, we would start the server once, at the beginning of the running container // Now, we start a new server each time we invoke a StartFunc. - stats.StartCancellableStatsServerWithPort(ctx, stats.DefaultStartupOptions(), serverHandlers) + if serverAdminHandlersWithStats() { + stats.StartCancellableStatsServerWithPort(ctx, stats.DefaultStartupOptions(), serverHandlers) + } else { + stats.StartCancellableStatsServerWithPort(ctx, stats.DefaultStartupOptions(), func(mux *http.ServeMux, profiles map[string]string) { + // let people know these moved + profiles[fmt.Sprintf("http://localhost:%d/snapshots/", AdminPort)] = fmt.Sprintf("To see snapshots, port forward to port %d", AdminPort) + }) + startHandlers(ctx, serverHandlers) + } return nil } } + +func startHandlers(ctx context.Context, addHandlers ...func(mux *http.ServeMux, profiles map[string]string)) error { + mux := new(http.ServeMux) + profileDescriptions := map[string]string{} + for _, addHandler := range addHandlers { + addHandler(mux, profileDescriptions) + } + idx := Index(profileDescriptions) + mux.HandleFunc("/", idx) + mux.HandleFunc("/snapshots/", idx) + server := &http.Server{ + Addr: fmt.Sprintf("localhost:%d", AdminPort), + Handler: mux, + } + contextutils.LoggerFrom(ctx).Infof("Admin server starting at %s", server.Addr) + go func() { + err := server.ListenAndServe() + if err == http.ErrServerClosed { + contextutils.LoggerFrom(ctx).Infof("Admin server closed") + } else { + contextutils.LoggerFrom(ctx).Warnf("Admin server closed with unexpected error: %v", err) + } + }() + go func() { + <-ctx.Done() + if server != nil { + err := server.Close() + contextutils.LoggerFrom(ctx).Warnf("Admin server shutdown returned error: %v", err) + } + }() + return nil +} + +func serverAdminHandlersWithStats() bool { + env := os.Getenv("ADMIN_HANDLERS_WITH_STATS") + return env == "true" +} + +func Index(profileDescriptions map[string]string) func(w http.ResponseWriter, r *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + + type profile struct { + Name string + Href string + Desc string + } + var profiles []profile + for href, desc := range profileDescriptions { + profiles = append(profiles, profile{ + Name: href, + Href: href, + Desc: desc, + }) + } + + sort.Slice(profiles, func(i, j int) bool { + return profiles[i].Name < profiles[j].Name + }) + + // Adding other profiles exposed from within this package + var buf bytes.Buffer + fmt.Fprintf(&buf, "

Admin Server

\n") + for _, p := range profiles { + fmt.Fprintf(&buf, "

%s

%s

\n", p.Name, p.Name, p.Desc) + + } + w.Write(buf.Bytes()) + } +} From 2311632020102354ae7741174c44e4f7128fd8e7 Mon Sep 17 00:00:00 2001 From: Yuval Kohavi Date: Fri, 8 Nov 2024 14:16:59 -0500 Subject: [PATCH 2/5] changelog --- changelog/v1.18.0-beta34/localhostdebug.yaml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changelog/v1.18.0-beta34/localhostdebug.yaml diff --git a/changelog/v1.18.0-beta34/localhostdebug.yaml b/changelog/v1.18.0-beta34/localhostdebug.yaml new file mode 100644 index 00000000000..833dff5193c --- /dev/null +++ b/changelog/v1.18.0-beta34/localhostdebug.yaml @@ -0,0 +1,3 @@ +changelog: + - type: NON_USER_FACING + description: Move snapshot debug endpoints localhost:9095. can be reverted by setting env ADMIN_HANDLERS_WITH_STATS to 'true' From 17f5de94d50cee06f2eaeb1987f4823a23ea82c0 Mon Sep 17 00:00:00 2001 From: Yuval Kohavi Date: Fri, 8 Nov 2024 16:53:45 -0500 Subject: [PATCH 3/5] fix test --- pkg/utils/glooadminutils/admincli/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/utils/glooadminutils/admincli/client.go b/pkg/utils/glooadminutils/admincli/client.go index 453a8c49190..d2b54d37131 100644 --- a/pkg/utils/glooadminutils/admincli/client.go +++ b/pkg/utils/glooadminutils/admincli/client.go @@ -14,7 +14,7 @@ import ( const ( InputSnapshotPath = "/snapshots/input" - DefaultAdminPort = 9091 + DefaultAdminPort = 9095 ) // Client is a utility for executing requests against the Gloo Admin API From be736ff8ecfe9006d1d9f51a7c61925622ece9ef Mon Sep 17 00:00:00 2001 From: Yuval Kohavi Date: Mon, 11 Nov 2024 14:37:15 -0500 Subject: [PATCH 4/5] PR comments --- pkg/utils/glooadminutils/admincli/client.go | 3 +- projects/gloo/pkg/servers/admin/server.go | 73 ++++++++++++++++++ projects/gloo/pkg/syncer/setup/start_func.go | 79 +------------------- 3 files changed, 79 insertions(+), 76 deletions(-) diff --git a/pkg/utils/glooadminutils/admincli/client.go b/pkg/utils/glooadminutils/admincli/client.go index d2b54d37131..11fb89b12ef 100644 --- a/pkg/utils/glooadminutils/admincli/client.go +++ b/pkg/utils/glooadminutils/admincli/client.go @@ -8,13 +8,14 @@ import ( "github.com/rotisserie/eris" "github.com/solo-io/gloo/pkg/utils/cmdutils" "github.com/solo-io/gloo/pkg/utils/requestutils/curl" + "github.com/solo-io/gloo/projects/gloo/pkg/servers/admin" "github.com/solo-io/go-utils/threadsafe" ) const ( InputSnapshotPath = "/snapshots/input" - DefaultAdminPort = 9095 + DefaultAdminPort = admin.AdminPort ) // Client is a utility for executing requests against the Gloo Admin API diff --git a/projects/gloo/pkg/servers/admin/server.go b/projects/gloo/pkg/servers/admin/server.go index eb02f063489..d298d9ebd0d 100644 --- a/projects/gloo/pkg/servers/admin/server.go +++ b/projects/gloo/pkg/servers/admin/server.go @@ -1,15 +1,22 @@ package admin import ( + "bytes" "context" "encoding/json" "fmt" "net/http" + "sort" "github.com/solo-io/gloo/projects/gloo/pkg/servers/iosnapshot" + "github.com/solo-io/go-utils/contextutils" "istio.io/istio/pkg/kube/krt" ) +const ( + AdminPort = 9095 +) + // ServerHandlers returns the custom handlers for the Admin Server, which will be bound to the http.ServeMux // These endpoints serve as the basis for an Admin Interface for the Control Plane (https://github.com/solo-io/gloo/issues/6494) func ServerHandlers(ctx context.Context, history iosnapshot.History, dbg *krt.DebugHandler) func(mux *http.ServeMux, profiles map[string]string) { @@ -98,3 +105,69 @@ func writeJSON(w http.ResponseWriter, obj any, req *http.Request) { w.WriteHeader(http.StatusInternalServerError) } } + +func StartHandlers(ctx context.Context, addHandlers ...func(mux *http.ServeMux, profiles map[string]string)) error { + mux := new(http.ServeMux) + profileDescriptions := map[string]string{} + for _, addHandler := range addHandlers { + addHandler(mux, profileDescriptions) + } + idx := index(profileDescriptions) + mux.HandleFunc("/", idx) + mux.HandleFunc("/snapshots/", idx) + server := &http.Server{ + Addr: fmt.Sprintf("localhost:%d", AdminPort), + Handler: mux, + } + contextutils.LoggerFrom(ctx).Infof("Admin server starting at %s", server.Addr) + go func() { + err := server.ListenAndServe() + if err == http.ErrServerClosed { + contextutils.LoggerFrom(ctx).Infof("Admin server closed") + } else { + contextutils.LoggerFrom(ctx).Warnf("Admin server closed with unexpected error: %v", err) + } + }() + go func() { + <-ctx.Done() + if server != nil { + err := server.Close() + if err != nil { + contextutils.LoggerFrom(ctx).Warnf("Admin server shutdown returned error: %v", err) + } + } + }() + return nil +} + +func index(profileDescriptions map[string]string) func(w http.ResponseWriter, r *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + + type profile struct { + Name string + Href string + Desc string + } + var profiles []profile + for href, desc := range profileDescriptions { + profiles = append(profiles, profile{ + Name: href, + Href: href, + Desc: desc, + }) + } + + sort.Slice(profiles, func(i, j int) bool { + return profiles[i].Name < profiles[j].Name + }) + + // Adding other profiles exposed from within this package + var buf bytes.Buffer + fmt.Fprintf(&buf, "

Admin Server

\n") + for _, p := range profiles { + fmt.Fprintf(&buf, "

%s

%s

\n", p.Name, p.Name, p.Desc) + + } + w.Write(buf.Bytes()) + } +} diff --git a/projects/gloo/pkg/syncer/setup/start_func.go b/projects/gloo/pkg/syncer/setup/start_func.go index 92ebd47439e..6cea9c0dd5d 100644 --- a/projects/gloo/pkg/syncer/setup/start_func.go +++ b/projects/gloo/pkg/syncer/setup/start_func.go @@ -1,13 +1,11 @@ package setup import ( - "bytes" "context" "fmt" "net/http" - "os" - "sort" + "github.com/solo-io/gloo/pkg/utils/envutils" "github.com/solo-io/gloo/projects/gloo/pkg/servers/admin" "github.com/solo-io/gloo/projects/gloo/pkg/servers/iosnapshot" "github.com/solo-io/go-utils/stats" @@ -20,10 +18,6 @@ import ( "github.com/solo-io/gloo/projects/gloo/pkg/bootstrap" ) -const ( - AdminPort = 9095 -) - // StartFunc represents a function that will be called with the initialized bootstrap.Opts // and Extensions. This is invoked each time the setup_syncer is executed // (which runs whenever the Setting CR is modified) @@ -77,80 +71,15 @@ func AdminServerStartFunc(history iosnapshot.History, dbg *krt.DebugHandler) Sta } else { stats.StartCancellableStatsServerWithPort(ctx, stats.DefaultStartupOptions(), func(mux *http.ServeMux, profiles map[string]string) { // let people know these moved - profiles[fmt.Sprintf("http://localhost:%d/snapshots/", AdminPort)] = fmt.Sprintf("To see snapshots, port forward to port %d", AdminPort) + profiles[fmt.Sprintf("http://localhost:%d/snapshots/", admin.AdminPort)] = fmt.Sprintf("To see snapshots, port forward to port %d", admin.AdminPort) }) - startHandlers(ctx, serverHandlers) + admin.StartHandlers(ctx, serverHandlers) } return nil } } -func startHandlers(ctx context.Context, addHandlers ...func(mux *http.ServeMux, profiles map[string]string)) error { - mux := new(http.ServeMux) - profileDescriptions := map[string]string{} - for _, addHandler := range addHandlers { - addHandler(mux, profileDescriptions) - } - idx := Index(profileDescriptions) - mux.HandleFunc("/", idx) - mux.HandleFunc("/snapshots/", idx) - server := &http.Server{ - Addr: fmt.Sprintf("localhost:%d", AdminPort), - Handler: mux, - } - contextutils.LoggerFrom(ctx).Infof("Admin server starting at %s", server.Addr) - go func() { - err := server.ListenAndServe() - if err == http.ErrServerClosed { - contextutils.LoggerFrom(ctx).Infof("Admin server closed") - } else { - contextutils.LoggerFrom(ctx).Warnf("Admin server closed with unexpected error: %v", err) - } - }() - go func() { - <-ctx.Done() - if server != nil { - err := server.Close() - contextutils.LoggerFrom(ctx).Warnf("Admin server shutdown returned error: %v", err) - } - }() - return nil -} - func serverAdminHandlersWithStats() bool { - env := os.Getenv("ADMIN_HANDLERS_WITH_STATS") - return env == "true" -} - -func Index(profileDescriptions map[string]string) func(w http.ResponseWriter, r *http.Request) { - return func(w http.ResponseWriter, r *http.Request) { - - type profile struct { - Name string - Href string - Desc string - } - var profiles []profile - for href, desc := range profileDescriptions { - profiles = append(profiles, profile{ - Name: href, - Href: href, - Desc: desc, - }) - } - - sort.Slice(profiles, func(i, j int) bool { - return profiles[i].Name < profiles[j].Name - }) - - // Adding other profiles exposed from within this package - var buf bytes.Buffer - fmt.Fprintf(&buf, "

Admin Server

\n") - for _, p := range profiles { - fmt.Fprintf(&buf, "

%s

%s

\n", p.Name, p.Name, p.Desc) - - } - w.Write(buf.Bytes()) - } + return envutils.IsEnvTruthy("ADMIN_HANDLERS_WITH_STATS") } From 1b27f44c36e96cbbfe14e3d2a4c6ae0a507ec401 Mon Sep 17 00:00:00 2001 From: Yuval Kohavi Date: Tue, 12 Nov 2024 01:28:24 -0500 Subject: [PATCH 5/5] pr comment --- pkg/utils/glooadminutils/admincli/client.go | 4 +--- test/kubernetes/testutils/assertions/gloo.go | 5 +++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/pkg/utils/glooadminutils/admincli/client.go b/pkg/utils/glooadminutils/admincli/client.go index 11fb89b12ef..f49d994082d 100644 --- a/pkg/utils/glooadminutils/admincli/client.go +++ b/pkg/utils/glooadminutils/admincli/client.go @@ -14,8 +14,6 @@ import ( const ( InputSnapshotPath = "/snapshots/input" - - DefaultAdminPort = admin.AdminPort ) // Client is a utility for executing requests against the Gloo Admin API @@ -35,7 +33,7 @@ func NewClient() *Client { curlOptions: []curl.Option{ curl.WithScheme("http"), curl.WithHost("127.0.0.1"), - curl.WithPort(DefaultAdminPort), + curl.WithPort(admin.AdminPort), // 3 retries, exponential back-off, 10 second max curl.WithRetries(3, 0, 10), }, diff --git a/test/kubernetes/testutils/assertions/gloo.go b/test/kubernetes/testutils/assertions/gloo.go index ece1af63452..be904c2db85 100644 --- a/test/kubernetes/testutils/assertions/gloo.go +++ b/test/kubernetes/testutils/assertions/gloo.go @@ -12,6 +12,7 @@ import ( "github.com/solo-io/gloo/pkg/utils/glooadminutils/admincli" "github.com/solo-io/gloo/pkg/utils/kubeutils/portforward" "github.com/solo-io/gloo/pkg/utils/requestutils/curl" + "github.com/solo-io/gloo/projects/gloo/pkg/servers/admin" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" ) @@ -26,7 +27,7 @@ func (p *Provider) AssertGlooAdminApi( portForwarder, err := p.clusterContext.Cli.StartPortForward(ctx, portforward.WithDeployment(glooDeployment.GetName(), glooDeployment.GetNamespace()), - portforward.WithPorts(int(admincli.DefaultAdminPort), int(admincli.DefaultAdminPort)), + portforward.WithPorts(int(admin.AdminPort), int(admin.AdminPort)), ) p.Require.NoError(err, "can open port-forward") defer func() { @@ -49,7 +50,7 @@ func (p *Provider) AssertGlooAdminApi( WithReceiver(io.Discard). // adminAssertion can overwrite this WithCurlOptions( curl.WithRetries(3, 0, 10), - curl.WithPort(int(admincli.DefaultAdminPort)), + curl.WithPort(int(admin.AdminPort)), ) for _, adminAssertion := range adminAssertions {