From 0843810f6ea7ed8d9b8292af59f4ceffb8f8c4fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zandr=C3=A9=20van=20Heerden?= Date: Tue, 16 Aug 2022 13:48:07 +0200 Subject: [PATCH] STAC-16788: Updated process agent graceful shutdown --- cmd/agent/collector.go | 15 ++++++------ cmd/agent/main_docker.go | 44 ++++++++++++++-------------------- cmd/agent/main_nodocker.go | 45 ++++++++++++++-------------------- cmd/network-tracer/main.go | 49 ++++++++++++++++---------------------- 4 files changed, 65 insertions(+), 88 deletions(-) diff --git a/cmd/agent/collector.go b/cmd/agent/collector.go index 0ba63753..0f6fa4b8 100644 --- a/cmd/agent/collector.go +++ b/cmd/agent/collector.go @@ -5,12 +5,6 @@ import ( "context" "encoding/json" "fmt" - "github.com/StackVista/stackstate-agent/pkg/aggregator" - "github.com/StackVista/stackstate-agent/pkg/batcher" - "github.com/StackVista/stackstate-agent/pkg/collector/check" - "github.com/StackVista/stackstate-agent/pkg/telemetry" - "github.com/StackVista/stackstate-agent/pkg/topology" - "github.com/StackVista/stackstate-process-agent/cmd/agent/features" "io" "io/ioutil" "math/rand" @@ -19,6 +13,13 @@ import ( "sync/atomic" "time" + "github.com/StackVista/stackstate-agent/pkg/aggregator" + "github.com/StackVista/stackstate-agent/pkg/batcher" + "github.com/StackVista/stackstate-agent/pkg/collector/check" + "github.com/StackVista/stackstate-agent/pkg/telemetry" + "github.com/StackVista/stackstate-agent/pkg/topology" + "github.com/StackVista/stackstate-process-agent/cmd/agent/features" + log "github.com/cihub/seelog" "github.com/StackVista/stackstate-process-agent/checks" @@ -140,7 +141,7 @@ func (l *Collector) run(exit chan bool) { } log.Infof("Starting process-agent for host=%s, endpoints=%s, enabled checks=%v", l.cfg.HostName, eps, l.cfg.EnabledChecks) - handleSignals(exit) + go HandleSignals(exit) heartbeat := time.NewTicker(15 * time.Second) queueSizeTicker := time.NewTicker(10 * time.Second) featuresTicker := time.NewTicker(5 * time.Second) diff --git a/cmd/agent/main_docker.go b/cmd/agent/main_docker.go index 4877fabf..2452b79b 100644 --- a/cmd/agent/main_docker.go +++ b/cmd/agent/main_docker.go @@ -12,31 +12,23 @@ import ( log "github.com/cihub/seelog" ) -// Handles signals - tells us whether we should exit. -func handleSignals(exit chan bool) { - signalCh := make(chan os.Signal, 1) - signal.Notify(signalCh, os.Interrupt, syscall.SIGTERM) - - go func() { - // Set up the signals async so we can Start the agent - select { - case sig := <-signalCh: - log.Infof("Received signal '%s', shutting down...", sig) - signalCh <- nil - exit <- true - default: - // continue - } - }() - - // By default systemd redirects the stdout to journald. When journald is stopped or crashes we receive a SIGPIPE signal. - // Go ignores SIGPIPE signals unless it is when stdout or stdout is closed, in this case the agent is stopped. - // We never want the agent to stop upon receiving SIGPIPE, so we intercept the SIGPIPE signals and just discard them. - sigpipeCh := make(chan os.Signal, 1) - signal.Notify(sigpipeCh, syscall.SIGPIPE) - go func() { - for range sigpipeCh { - // do nothing +// HandleSignals tells us whether we should exit. +func HandleSignals(exit chan struct{}) { + sigIn := make(chan os.Signal, 100) + signal.Notify(sigIn, syscall.SIGINT, syscall.SIGTERM, syscall.SIGPIPE) + // unix only in all likelihood; but we don't care. + for sig := range sigIn { + switch sig { + case syscall.SIGINT, syscall.SIGTERM: + log.Infof("Caught signal '%s'; terminating.", sig) + close(exit) + return + case syscall.SIGPIPE: + // By default systemd redirects the stdout to journald. When journald is stopped or crashes we receive a SIGPIPE signal. + // Go ignores SIGPIPE signals unless it is when stderr or stdout is closed, in this case the agent is stopped. + // We never want the agent to stop upon receiving SIGPIPE, so we intercept the SIGPIPE signals and just discard them. + // See https://golang.org/pkg/os/signal/#hdr-SIGPIPE + continue } - }() + } } diff --git a/cmd/agent/main_nodocker.go b/cmd/agent/main_nodocker.go index 47434bac..926334c4 100644 --- a/cmd/agent/main_nodocker.go +++ b/cmd/agent/main_nodocker.go @@ -11,32 +11,23 @@ import ( log "github.com/cihub/seelog" ) -// Handles signals - tells us whether we should exit. -func handleSignals(exit chan bool) { - signalCh := make(chan os.Signal, 1) - signal.Notify(signalCh, os.Interrupt, syscall.SIGTERM) - - go func() { - // Set up the signals async so we can Start the agent - select { - case sig := <-signalCh: - log.Infof("Received signal '%s', shutting down...", sig) - signalCh <- nil - exit <- true - default: - // continue - } - }() - - // By default systemd redirects the stdout to journald. When journald is stopped or crashes we receive a SIGPIPE signal. - // Go ignores SIGPIPE signals unless it is when stdout or stdout is closed, in this case the agent is stopped. - // We never want the agent to stop upon receiving SIGPIPE, so we intercept the SIGPIPE signals and just discard them. - sigpipeCh := make(chan os.Signal, 1) - signal.Notify(sigpipeCh, syscall.SIGPIPE) - go func() { - for range sigpipeCh { - // do nothing +// HandleSignals tells us whether we should exit. +func HandleSignals(exit chan bool) { + sigIn := make(chan os.Signal, 100) + signal.Notify(sigIn, syscall.SIGINT, syscall.SIGTERM, syscall.SIGPIPE) + // unix only in all likelihood; but we don't care. + for sig := range sigIn { + switch sig { + case syscall.SIGINT, syscall.SIGTERM: + log.Infof("Caught signal '%s'; terminating.", sig) + close(exit) + return + case syscall.SIGPIPE: + // By default systemd redirects the stdout to journald. When journald is stopped or crashes we receive a SIGPIPE signal. + // Go ignores SIGPIPE signals unless it is when stderr or stdout is closed, in this case the agent is stopped. + // We never want the agent to stop upon receiving SIGPIPE, so we intercept the SIGPIPE signals and just discard them. + // See https://golang.org/pkg/os/signal/#hdr-SIGPIPE + continue } - }() - + } } diff --git a/cmd/network-tracer/main.go b/cmd/network-tracer/main.go index 4f2deed6..2c0d0505 100644 --- a/cmd/network-tracer/main.go +++ b/cmd/network-tracer/main.go @@ -92,9 +92,9 @@ func main() { log.Infof("network tracer started") // Handles signals, which tells us whether we should exit. - e := make(chan bool) - handleSignals(e) - <-e + exit := make(chan bool) + go HandleSignals(exit) + <-exit } func gracefulExit() { @@ -105,32 +105,25 @@ func gracefulExit() { os.Exit(0) } -func handleSignals(exit chan bool) { - signalCh := make(chan os.Signal, 1) - signal.Notify(signalCh, os.Interrupt, syscall.SIGTERM) - - go func() { - // Set up the signals async so we can Start the agent - select { - case sig := <-signalCh: - log.Infof("Received signal '%s', shutting down...", sig) - signalCh <- nil - exit <- true - default: - // continue +// HandleSignals tells us whether we should exit. +func HandleSignals(exit chan bool) { + sigIn := make(chan os.Signal, 100) + signal.Notify(sigIn, syscall.SIGINT, syscall.SIGTERM, syscall.SIGPIPE) + // unix only in all likelihood; but we don't care. + for sig := range sigIn { + switch sig { + case syscall.SIGINT, syscall.SIGTERM: + log.Infof("Caught signal '%s'; terminating.", sig) + close(exit) + return + case syscall.SIGPIPE: + // By default systemd redirects the stdout to journald. When journald is stopped or crashes we receive a SIGPIPE signal. + // Go ignores SIGPIPE signals unless it is when stderr or stdout is closed, in this case the agent is stopped. + // We never want the agent to stop upon receiving SIGPIPE, so we intercept the SIGPIPE signals and just discard them. + // See https://golang.org/pkg/os/signal/#hdr-SIGPIPE + continue } - }() - - // By default systemd redirects the stdout to journald. When journald is stopped or crashes we receive a SIGPIPE signal. - // Go ignores SIGPIPE signals unless it is when stdout or stdout is closed, in this case the agent is stopped. - // We never want the agent to stop upon receiving SIGPIPE, so we intercept the SIGPIPE signals and just discard them. - sigpipeCh := make(chan os.Signal, 1) - signal.Notify(sigpipeCh, syscall.SIGPIPE) - go func() { - for range sigpipeCh { - // do nothing - } - }() + } } // versionString returns the version information filled in at build time