STAC-16788: Updated process agent graceful shutdown

cmd/agent/collector.go

@@ -5,12 +5,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"github.com/StackVista/stackstate-agent/pkg/aggregator"
-	"github.com/StackVista/stackstate-agent/pkg/batcher"
-	"github.com/StackVista/stackstate-agent/pkg/collector/check"
-	"github.com/StackVista/stackstate-agent/pkg/telemetry"
-	"github.com/StackVista/stackstate-agent/pkg/topology"
-	"github.com/StackVista/stackstate-process-agent/cmd/agent/features"
 	"io"
 	"io/ioutil"
 	"math/rand"
@@ -19,6 +13,13 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/StackVista/stackstate-agent/pkg/aggregator"
+	"github.com/StackVista/stackstate-agent/pkg/batcher"
+	"github.com/StackVista/stackstate-agent/pkg/collector/check"
+	"github.com/StackVista/stackstate-agent/pkg/telemetry"
+	"github.com/StackVista/stackstate-agent/pkg/topology"
+	"github.com/StackVista/stackstate-process-agent/cmd/agent/features"
+
 	log "github.com/cihub/seelog"
 
 	"github.com/StackVista/stackstate-process-agent/checks"
@@ -140,7 +141,7 @@ func (l *Collector) run(exit chan bool) {
 	}
 	log.Infof("Starting process-agent for host=%s, endpoints=%s, enabled checks=%v", l.cfg.HostName, eps, l.cfg.EnabledChecks)
 
-	handleSignals(exit)
+	go HandleSignals(exit)
 	heartbeat := time.NewTicker(15 * time.Second)
 	queueSizeTicker := time.NewTicker(10 * time.Second)
 	featuresTicker := time.NewTicker(5 * time.Second)

cmd/agent/main_docker.go

@@ -12,31 +12,23 @@ import (
 	log "github.com/cihub/seelog"
 )
 
-// Handles signals - tells us whether we should exit.
-func handleSignals(exit chan bool) {
-	signalCh := make(chan os.Signal, 1)
-	signal.Notify(signalCh, os.Interrupt, syscall.SIGTERM)
-
-	go func() {
-		// Set up the signals async so we can Start the agent
-		select {
-		case sig := <-signalCh:
-			log.Infof("Received signal '%s', shutting down...", sig)
-			signalCh <- nil
-			exit <- true
-		default:
-			// continue
-		}
-	}()
-
-	// By default systemd redirects the stdout to journald. When journald is stopped or crashes we receive a SIGPIPE signal.
-	// Go ignores SIGPIPE signals unless it is when stdout or stdout is closed, in this case the agent is stopped.
-	// We never want the agent to stop upon receiving SIGPIPE, so we intercept the SIGPIPE signals and just discard them.
-	sigpipeCh := make(chan os.Signal, 1)
-	signal.Notify(sigpipeCh, syscall.SIGPIPE)
-	go func() {
-		for range sigpipeCh {
-			// do nothing
+// HandleSignals tells us whether we should exit.
+func HandleSignals(exit chan struct{}) {
+	sigIn := make(chan os.Signal, 100)
+	signal.Notify(sigIn, syscall.SIGINT, syscall.SIGTERM, syscall.SIGPIPE)
+	// unix only in all likelihood; but we don't care.
+	for sig := range sigIn {
+		switch sig {
+		case syscall.SIGINT, syscall.SIGTERM:
+			log.Infof("Caught signal '%s'; terminating.", sig)
+			close(exit)
+			return
+		case syscall.SIGPIPE:
+			// By default systemd redirects the stdout to journald. When journald is stopped or crashes we receive a SIGPIPE signal.
+			// Go ignores SIGPIPE signals unless it is when stderr or stdout is closed, in this case the agent is stopped.
+			// We never want the agent to stop upon receiving SIGPIPE, so we intercept the SIGPIPE signals and just discard them.
+			// See https://golang.org/pkg/os/signal/#hdr-SIGPIPE
+			continue
 		}
-	}()
+	}
 }

cmd/agent/main_nodocker.go

@@ -11,32 +11,23 @@ import (
 	log "github.com/cihub/seelog"
 )
 
-// Handles signals - tells us whether we should exit.
-func handleSignals(exit chan bool) {
-	signalCh := make(chan os.Signal, 1)
-	signal.Notify(signalCh, os.Interrupt, syscall.SIGTERM)
-
-	go func() {
-		// Set up the signals async so we can Start the agent
-		select {
-		case sig := <-signalCh:
-			log.Infof("Received signal '%s', shutting down...", sig)
-			signalCh <- nil
-			exit <- true
-		default:
-			// continue
-		}
-	}()
-
-	// By default systemd redirects the stdout to journald. When journald is stopped or crashes we receive a SIGPIPE signal.
-	// Go ignores SIGPIPE signals unless it is when stdout or stdout is closed, in this case the agent is stopped.
-	// We never want the agent to stop upon receiving SIGPIPE, so we intercept the SIGPIPE signals and just discard them.
-	sigpipeCh := make(chan os.Signal, 1)
-	signal.Notify(sigpipeCh, syscall.SIGPIPE)
-	go func() {
-		for range sigpipeCh {
-			// do nothing
+// HandleSignals tells us whether we should exit.
+func HandleSignals(exit chan bool) {
+	sigIn := make(chan os.Signal, 100)
+	signal.Notify(sigIn, syscall.SIGINT, syscall.SIGTERM, syscall.SIGPIPE)
+	// unix only in all likelihood; but we don't care.
+	for sig := range sigIn {
+		switch sig {
+		case syscall.SIGINT, syscall.SIGTERM:
+			log.Infof("Caught signal '%s'; terminating.", sig)
+			close(exit)
+			return
+		case syscall.SIGPIPE:
+			// By default systemd redirects the stdout to journald. When journald is stopped or crashes we receive a SIGPIPE signal.
+			// Go ignores SIGPIPE signals unless it is when stderr or stdout is closed, in this case the agent is stopped.
+			// We never want the agent to stop upon receiving SIGPIPE, so we intercept the SIGPIPE signals and just discard them.
+			// See https://golang.org/pkg/os/signal/#hdr-SIGPIPE
+			continue
 		}
-	}()
-
+	}
 }

cmd/network-tracer/main.go

@@ -92,9 +92,9 @@ func main() {
 	log.Infof("network tracer started")
 
 	// Handles signals, which tells us whether we should exit.
-	e := make(chan bool)
-	handleSignals(e)
-	<-e
+	exit := make(chan bool)
+	go HandleSignals(exit)
+	<-exit
 }
 
 func gracefulExit() {
@@ -105,32 +105,25 @@ func gracefulExit() {
 	os.Exit(0)
 }
 
-func handleSignals(exit chan bool) {
-	signalCh := make(chan os.Signal, 1)
-	signal.Notify(signalCh, os.Interrupt, syscall.SIGTERM)
-
-	go func() {
-		// Set up the signals async so we can Start the agent
-		select {
-		case sig := <-signalCh:
-			log.Infof("Received signal '%s', shutting down...", sig)
-			signalCh <- nil
-			exit <- true
-		default:
-			// continue
+// HandleSignals tells us whether we should exit.
+func HandleSignals(exit chan bool) {
+	sigIn := make(chan os.Signal, 100)
+	signal.Notify(sigIn, syscall.SIGINT, syscall.SIGTERM, syscall.SIGPIPE)
+	// unix only in all likelihood; but we don't care.
+	for sig := range sigIn {
+		switch sig {
+		case syscall.SIGINT, syscall.SIGTERM:
+			log.Infof("Caught signal '%s'; terminating.", sig)
+			close(exit)
+			return
+		case syscall.SIGPIPE:
+			// By default systemd redirects the stdout to journald. When journald is stopped or crashes we receive a SIGPIPE signal.
+			// Go ignores SIGPIPE signals unless it is when stderr or stdout is closed, in this case the agent is stopped.
+			// We never want the agent to stop upon receiving SIGPIPE, so we intercept the SIGPIPE signals and just discard them.
+			// See https://golang.org/pkg/os/signal/#hdr-SIGPIPE
+			continue
 		}
-	}()
-
-	// By default systemd redirects the stdout to journald. When journald is stopped or crashes we receive a SIGPIPE signal.
-	// Go ignores SIGPIPE signals unless it is when stdout or stdout is closed, in this case the agent is stopped.
-	// We never want the agent to stop upon receiving SIGPIPE, so we intercept the SIGPIPE signals and just discard them.
-	sigpipeCh := make(chan os.Signal, 1)
-	signal.Notify(sigpipeCh, syscall.SIGPIPE)
-	go func() {
-		for range sigpipeCh {
-			// do nothing
-		}
-	}()
+	}
 }
 
 // versionString returns the version information filled in at build time