Guest shutdown support in endpointVM

This adds neat shutdown support in the endpointVM:
* cleans up Views, Managers and sessions in portlayer
* cleans up session in personality
* cleans up and purges all active sessions in vicadmin

Updates the way tether waits for sessions to exit.
Updates vic-machine to use guest shutdown for endpointVM

cmd/docker/main.go

@@ -15,6 +15,7 @@
 package main
 
 import (
+	"context"
 	"crypto/tls"
 	"crypto/x509"
 	"flag"
@@ -122,12 +123,16 @@ func main() {
 	serveAPIWait := make(chan error)
 	go api.Wait(serveAPIWait)
 
+	// signal.Trap explicitly calls os.Exit so any exit logic has to be rolled in here
 	signal.Trap(func() {
+		log.Info("Closing down docker personality")
+
 		api.Close()
+		plEventMonitor.Stop()
+		vicbackends.Finalize(context.Background())
 	})
 
 	<-serveAPIWait
-	plEventMonitor.Stop()
 }
 
 func handleFlags() bool {

cmd/port-layer-server/main.go

@@ -149,6 +149,9 @@ func main() {
 	go func() {
 		<-sig
 
+		// if vspc server fails to shut down in a timely manner then the clean endpointVM shutdown will fail
+		// this failure to shut down cleanly can be observed in the "/var/log/vic/init.log" or
+		// "[datastore] endpointVM/tether.debug" log files as vic-init will report that it's waiting for the portlayer to exit.
 		vspc.Stop()
 		dnsserver.Stop()
 		restapi.StopAPIServers()

cmd/tether/main_linux.go

@@ -121,16 +121,30 @@ func halt() {
 	syscall.Reboot(syscall.LINUX_REBOOT_CMD_POWER_OFF)
 }
 
+// This code is mirrored in cmd/vic-init/main_linux.go and should be de-duped
 func startSignalHandler() {
 	sigs := make(chan os.Signal, 1)
-	signal.Notify(sigs, syscall.SIGHUP)
+	// currently using a really basic set of signal behaviours, reflected from here:
+	//    https://github.com/troglobit/finit/blob/master/docs/signals.md
+	// itself sourced from here:
+	//    https://unix.stackexchange.com/a/191875
+	// This set was chosen because of suitably limited scope vs the complexity of systemd or similar.
+	signal.Notify(sigs, syscall.SIGHUP, syscall.SIGUSR1, syscall.SIGUSR2, syscall.SIGPWR, syscall.SIGTERM, syscall.SIGINT)
 
 	go func() {
 		for s := range sigs {
 			switch s {
 			case syscall.SIGHUP:
-				log.Infof("Reloading tether configuration")
+				log.Infof("Reloading tether configuration via signal %s", s.String())
 				tthr.Reload()
+			case syscall.SIGUSR1, syscall.SIGUSR2, syscall.SIGPWR:
+				log.Infof("Stopping tether via signal %s", s.String())
+				tthr.Stop()
+			case syscall.SIGTERM, syscall.SIGINT:
+				log.Infof("Stopping system in lieu of restart handling via signal %s", s.String())
+				// TODO: update this to adjust power off handling for reboot
+				// this should be in guest reboot rather than power cycle
+				tthr.Stop()
 			default:
 				log.Infof("%s signal not defined", s.String())
 			}

cmd/tether/main_test.go

@@ -135,6 +135,10 @@ func (m *mockery) Stop() error {
 	return nil
 }
 
+func (m *mockery) Wait(ctx context.Context) error {
+	return nil
+}
+
 func (m *mockery) Register(name string, config tether.Extension) {
 }
 

cmd/vic-init/main_linux.go

@@ -15,7 +15,10 @@
 package main
 
 import (
+	"context"
+	"errors"
 	"os"
+	"os/signal"
 	"runtime/debug"
 	"syscall"
 	"time"
@@ -25,6 +28,7 @@ import (
 
 	"github.com/vmware/govmomi/toolbox"
 	"github.com/vmware/vic/lib/tether"
+	"github.com/vmware/vic/lib/tether/shared"
 	viclog "github.com/vmware/vic/pkg/log"
 	"github.com/vmware/vic/pkg/log/syslog"
 	"github.com/vmware/vic/pkg/logmgr"
@@ -79,6 +83,8 @@ func main() {
 	extraconfig.Decode(src, &config)
 	debugLevel = config.Diagnostics.DebugLevel
 
+	startSignalHandler()
+
 	logcfg := viclog.NewLoggingConfig()
 	if debugLevel > 0 {
 		logcfg.Level = log.DebugLevel
@@ -136,33 +142,80 @@ func main() {
 	log.Info("Clean exit from init")
 }
 
+// exitTether signals the current process, which triggers tether.Stop and the killing of its children.
+// NOTE: I don't like having this here and it really needs to be moved into an interface that
+// can be provided to toolbox for system callbacks. While this could be part of the Operations
+// interface I think I'd rather have a separate one specifically for the possible toolbox interactions.
+func exitTether() error {
+	defer trace.End(trace.Begin(""))
+
+	p, err := os.FindProcess(os.Getpid())
+	if err != nil {
+		return err
+	}
+
+	if err = p.Signal(syscall.SIGUSR2); err != nil {
+		return err
+	}
+
+	return err
+}
+
 // exit cleanly shuts down the system
-func halt() {
+func halt() error {
 	log.Infof("Powering off the system")
-	if debugLevel > 0 {
+
+	err := exitTether()
+	if err != nil {
+		log.Warn(err)
+	}
+
+	if debugLevel > 2 {
 		log.Info("Squashing power off for debug init")
-		return
+		return errors.New("debug config suppresses shutdown")
 	}
 
+	timeout, cancel := context.WithTimeout(context.Background(), shared.GuestShutdownTimeout)
+	err = tthr.Wait(timeout)
+	cancel()
+
 	syscall.Sync()
 	syscall.Reboot(syscall.LINUX_REBOOT_CMD_POWER_OFF)
+
+	return err
 }
 
-func reboot() {
+func reboot() error {
 	log.Infof("Rebooting the system")
-	if debugLevel > 0 {
+
+	err := exitTether()
+	if err != nil {
+		log.Warn(err)
+	}
+
+	if debugLevel > 2 {
 		log.Info("Squashing reboot for debug init")
-		return
+		return errors.New("debug config suppresses reboot")
 	}
 
+	timeout, cancel := context.WithTimeout(context.Background(), shared.GuestRebootTimeout)
+	err = tthr.Wait(timeout)
+	cancel()
+
 	syscall.Sync()
 	syscall.Reboot(syscall.LINUX_REBOOT_CMD_RESTART)
+
+	return err
 }
 
 func configureToolbox(t *tether.Toolbox) *tether.Toolbox {
 	cmd := t.Service.Command
 	cmd.ProcessStartCommand = startCommand
 
+	t.Power.Halt.Handler = halt
+	t.Power.Reboot.Handler = reboot
+	t.Power.Suspend.Handler = exitTether
+
 	return t
 }
 
@@ -201,3 +254,31 @@ func defaultIP() string {
 
 	return toolbox.DefaultIP()
 }
+
+// This code is mirrored in cmd/tether/main_linux.go and should be de-duped
+func startSignalHandler() {
+	sigs := make(chan os.Signal, 1)
+	signal.Notify(sigs, syscall.SIGHUP, syscall.SIGUSR1, syscall.SIGUSR2, syscall.SIGPWR, syscall.SIGTERM, syscall.SIGINT)
+
+	go func() {
+		for s := range sigs {
+			switch s {
+			case syscall.SIGHUP:
+				log.Infof("Reloading tether configuration")
+				tthr.Reload()
+			case syscall.SIGUSR1, syscall.SIGUSR2, syscall.SIGPWR:
+				log.Infof("Stopping tether via signal %s", s.String())
+				tthr.Stop()
+				return
+			case syscall.SIGTERM, syscall.SIGINT:
+				log.Infof("Stopping system in lieu of restart handling via signal %s", s.String())
+				// TODO: update this to adjust power off handling for reboot
+				// this should be in guest reboot rather than power cycle
+				tthr.Stop()
+				return
+			default:
+				log.Infof("%s signal not defined", s.String())
+			}
+		}
+	}()
+}

cmd/vicadmin/server.go

@@ -444,6 +444,8 @@ func (s *server) serve() error {
 func (s *server) stop() error {
 	defer trace.End(trace.Begin(""))
 
+	s.uss.Destroy()
+
 	if s.l != nil {
 		err := s.l.Close()
 		s.l = nil

cmd/vicadmin/usersession.go

@@ -69,6 +69,13 @@ func (u *UserSessionStore) Add(id string, config *session.Config, vs *session.Se
 func (u *UserSessionStore) Delete(id string) {
 	u.mutex.Lock()
 	defer u.mutex.Unlock()
+
+	us := u.sessions[id]
+	if us != nil && us.vsphere != nil {
+		log.Infof("Logging out vSphere session for %s", id)
+		us.vsphere.Logout(context.Background())
+	}
+
 	delete(u.sessions, id)
 }
 
@@ -112,6 +119,13 @@ func (u *UserSessionStore) reaper() {
 	}
 }
 
+// Destroy will logout and delete all sessions in the store, irrespective of expiration
+func (u *UserSessionStore) Destroy() {
+	for id := range u.sessions {
+		u.Delete(id)
+	}
+}
+
 // NewUserSessionStore creates & initializes a UserSessionStore and starts a session reaper in the background
 func NewUserSessionStore() *UserSessionStore {
 	u := &UserSessionStore{

cmd/vicadmin/vicadm.go

@@ -613,15 +613,9 @@ func main() {
 	}
 
 	log.Infof("listening on %s", s.addr)
-	signals := []syscall.Signal{
-		syscall.SIGTERM,
-		syscall.SIGINT,
-	}
 
 	sigchan := make(chan os.Signal, 1)
-	for _, signum := range signals {
-		signal.Notify(sigchan, signum)
-	}
+	signal.Notify(sigchan, syscall.SIGTERM, syscall.SIGINT)
 
 	go func() {
 		signal := <-sigchan

infra/scripts/replace-running-components.sh

@@ -152,7 +152,7 @@ function replace-component() {
     if [[ $1 == "vic-init" ]]; then
         on-vch systemctl restart vic-init
     else
-        on-vch kill -9 $pid
+        on-vch kill -TERM $pid
     fi
 }
 

lib/apiservers/engine/backends/backends.go

@@ -154,6 +154,18 @@ func Init(portLayerAddr, product string, port uint, config *config.VirtualContai
 	return nil
 }
 
+// Finalize performs cleanup before a graceful exit of the API server.
+// In this case that means logging out the dynamic config session if any.
+func Finalize(ctx context.Context) error {
+	log.Info("Shutting down docker API server backend")
+
+	if vchConfig != nil && vchConfig.sess != nil {
+		vchConfig.sess.Logout(ctx)
+	}
+
+	return nil
+}
+
 func hydrateCaches(op trace.Operation) error {
 	const waiters = 3
 

lib/apiservers/portlayer/restapi/configure_port_layer.go

@@ -34,7 +34,6 @@ import (
 	"github.com/vmware/vic/lib/apiservers/portlayer/restapi/operations"
 	"github.com/vmware/vic/lib/apiservers/portlayer/restapi/options"
 	"github.com/vmware/vic/lib/portlayer"
-	"github.com/vmware/vic/lib/portlayer/exec"
 	"github.com/vmware/vic/pkg/version"
 	"github.com/vmware/vic/pkg/vsphere/session"
 )
@@ -59,7 +58,7 @@ var portlayerhandlers = []handler{
 
 var apiServers []*graceful.Server
 
-const stopTimeout = time.Second * 3
+const stopTimeout = time.Second * 20
 
 func configureFlags(api *operations.PortLayerAPI) {
 	api.CommandLineOptionsGroups = []swag.CommandLineOptionsGroup{
@@ -96,11 +95,8 @@ func configureAPI(api *operations.PortLayerAPI) http.Handler {
 	api.ServerShutdown = func() {
 		log.Infof("Shutting down port-layer-server")
 
-		// stop the event collectors
-		collectors := exec.Config.EventManager.Collectors()
-		for _, c := range collectors {
-			c.Stop()
-		}
+		// shut down any component specific long running daemon tasks
+		portlayer.Finalize(context.Background())
 
 		// Logout the session
 		if err := sess.Logout(ctx); err != nil {
@@ -110,6 +106,8 @@ func configureAPI(api *operations.PortLayerAPI) http.Handler {
 
 	// initialize the port layer
 	if err = portlayer.Init(ctx, sess); err != nil {
+		// ensure that any setup that was done is unwound neatly
+		portlayer.Finalize(context.Background())
 		log.Fatalf("could not initialize port layer: %s", err)
 	}