From 3344856009038f2b2d6daff2ae4f9554cccbfd64 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Thu, 11 May 2023 05:46:33 -0400 Subject: [PATCH] Add support to route both GRPC and HTTP over the HTTP server (#288) * add support for grpc on the http port Signed-off-by: Joe Elliott --- go.mod | 1 + go.sum | 3 ++ server/server.go | 78 ++++++++++++++++++++++++++++++++++++------------ 3 files changed, 63 insertions(+), 19 deletions(-) diff --git a/go.mod b/go.mod index bb6f79a0..71308a21 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,7 @@ require ( github.com/prometheus/exporter-toolkit v0.8.2 github.com/sercand/kuberesolver/v4 v4.0.0 github.com/sirupsen/logrus v1.6.0 + github.com/soheilhy/cmux v0.1.5 // indirect github.com/stretchr/testify v1.8.1 github.com/uber/jaeger-client-go v2.28.0+incompatible github.com/uber/jaeger-lib v2.2.0+incompatible diff --git a/go.sum b/go.sum index 15999095..080336b2 100644 --- a/go.sum +++ b/go.sum @@ -668,6 +668,8 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= +github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= @@ -794,6 +796,7 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= diff --git a/server/server.go b/server/server.go index 5442c12d..4026516f 100644 --- a/server/server.go +++ b/server/server.go @@ -17,6 +17,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/prometheus/exporter-toolkit/web" + "github.com/soheilhy/cmux" "golang.org/x/net/context" "golang.org/x/net/netutil" "google.golang.org/grpc" @@ -96,6 +97,7 @@ type Config struct { HTTPMiddleware []middleware.Interface `yaml:"-"` Router *mux.Router `yaml:"-"` DoNotAddDefaultHTTPMiddleware bool `yaml:"-"` + RouteHTTPToGRPC bool `yaml:"-"` GPRCServerMaxRecvMsgSize int `yaml:"grpc_server_max_recv_msg_size"` GRPCServerMaxSendMsgSize int `yaml:"grpc_server_max_send_msg_size"` @@ -185,6 +187,13 @@ type Server struct { grpcListener net.Listener httpListener net.Listener + // These fields are used to support grpc over the http server + // if RouteHTTPToGRPC is set. the fields are kept here + // so they can be initialized in New() and started in Run() + grpchttpmux cmux.CMux + grpcOnHTTPListener net.Listener + GRPCOnHTTPServer *grpc.Server + HTTP *mux.Router HTTPServer *http.Server GRPC *grpc.Server @@ -242,6 +251,15 @@ func New(cfg Config) (*Server, error) { httpListener = netutil.LimitListener(httpListener, cfg.HTTPConnLimit) } + var grpcOnHTTPListener net.Listener + var grpchttpmux cmux.CMux + if cfg.RouteHTTPToGRPC { + grpchttpmux = cmux.New(httpListener) + + httpListener = grpchttpmux.Match(cmux.HTTP1Fast()) + grpcOnHTTPListener = grpchttpmux.Match(cmux.HTTP2()) + } + network = cfg.GRPCListenNetwork if network == "" { network = DefaultNetwork @@ -384,6 +402,7 @@ func New(cfg Config) (*Server, error) { grpcOptions = append(grpcOptions, grpc.Creds(grpcCreds)) } grpcServer := grpc.NewServer(grpcOptions...) + grpcOnHttpServer := grpc.NewServer(grpcOptions...) // Setup HTTP server var router *mux.Router @@ -446,17 +465,20 @@ func New(cfg Config) (*Server, error) { } return &Server{ - cfg: cfg, - httpListener: httpListener, - grpcListener: grpcListener, - handler: handler, - - HTTP: router, - HTTPServer: httpServer, - GRPC: grpcServer, - Log: log, - Registerer: reg, - Gatherer: gatherer, + cfg: cfg, + httpListener: httpListener, + grpcListener: grpcListener, + grpcOnHTTPListener: grpcOnHTTPListener, + handler: handler, + grpchttpmux: grpchttpmux, + + HTTP: router, + HTTPServer: httpServer, + GRPC: grpcServer, + GRPCOnHTTPServer: grpcOnHttpServer, + Log: log, + Registerer: reg, + Gatherer: gatherer, }, nil } @@ -509,19 +531,37 @@ func (s *Server) Run() error { go func() { err := s.GRPC.Serve(s.grpcListener) - if err == grpc.ErrServerStopped { - err = nil - } - - select { - case errChan <- err: - default: - } + handleGRPCError(err, errChan) }() + // grpchttpmux will only be set if grpchttpmux RouteHTTPToGRPC is set + if s.grpchttpmux != nil { + go func() { + err := s.grpchttpmux.Serve() + handleGRPCError(err, errChan) + }() + go func() { + err := s.GRPCOnHTTPServer.Serve(s.grpcOnHTTPListener) + handleGRPCError(err, errChan) + }() + } + return <-errChan } +// handleGRPCError consolidates GRPC Server error handling by sending +// any error to errChan except for grpc.ErrServerStopped which is ignored. +func handleGRPCError(err error, errChan chan error) { + if err == grpc.ErrServerStopped { + err = nil + } + + select { + case errChan <- err: + default: + } +} + // HTTPListenAddr exposes `net.Addr` that `Server` is listening to for HTTP connections. func (s *Server) HTTPListenAddr() net.Addr { return s.httpListener.Addr()