Use systemd as service manager when --service-mgr argument is passed

extra/kadalu-mgr.service

@@ -4,7 +4,7 @@ After=network.target
 
 [Service]
 PIDFile=/var/run/kadalu/kadalu-mgr.pid
-ExecStart=/usr/sbin/kadalu mgr --workdir=/var/lib/kadalu --logdir=/var/log/kadalu
+ExecStart=/usr/sbin/kadalu mgr --workdir=/var/lib/kadalu --logdir=/var/log/kadalu --service-mgr=systemd
 
 [Install]
 WantedBy=multi-user.target

mgr/src/cmds/mgr.cr

@@ -5,7 +5,8 @@ struct MgrArgs
   property metrics_interval_seconds = 15,
     workdir = "/var/lib/kadalu",
     logdir = "",
-    hostname = ""
+    hostname = "",
+    service_mgr = ""
 end
 
 class Args
@@ -23,6 +24,9 @@ command "mgr", "Start the kadalu storage manager" do |parser, args|
   parser.on("--hostname=HOSTNAME", "Set hostname") do |hostname|
     args.mgr_args.hostname = hostname
   end
+  parser.on("--service-mgr=SVC_MANAGER", "Service manager (systemd,supervisord,..). Default is None") do |svc_mgr|
+    args.mgr_args.service_mgr = svc_mgr
+  end
 end
 
 handler "mgr" do |args|

mgr/src/server/conf.cr

@@ -7,7 +7,8 @@ class GlobalConfig
     pool_name = "",
     local_hostname : String = `hostname`.strip,
     local_node = LocalNodeData.new,
-    agent = false
+    agent = false,
+    service_mgr = ""
 end
 
 struct LocalNodeData

mgr/src/server/plugins/volume_utils.cr

@@ -142,14 +142,14 @@ def restart_shd_service_and_manage_rebalance_services(services, volume_name, act
     services[GlobalConfig.local_node.id].each do |service|
       svc = Service.from_json(service.to_json)
       if svc.name == "shdservice"
-        svc.restart
+        svc.restart(plugin: GlobalConfig.service_mgr)
       elsif svc.name == "fixlayoutservice" || svc.name == "migratedataservice"
         status_file_path = "/var/lib/kadalu/rebalance/#{volume_name}/#{svc.id}.json"
         FileUtils.rm(status_file_path) if File.exists?(status_file_path)
         if action == "start"
-          svc.start
+          svc.start(plugin: GlobalConfig.service_mgr)
         else
-          svc.stop
+          svc.stop(plugin: GlobalConfig.service_mgr)
         end
       end
     end
@@ -175,9 +175,9 @@ def handle_node_volume_start_stop(data, action)
     services[GlobalConfig.local_node.id].each do |service|
       svc = Service.from_json(service.to_json)
       if action == "start"
-        svc.start
+        svc.start(plugin: GlobalConfig.service_mgr)
       else
-        svc.stop
+        svc.stop(plugin: GlobalConfig.service_mgr)
       end
     end
   end
@@ -240,7 +240,7 @@ def handle_volume_create(data, stopped = false)
     Dir.mkdir_p("/run/kadalu")
     services[GlobalConfig.local_node.id].each do |service|
       svc = Service.from_json(service.to_json)
-      svc.start
+      svc.start(plugin: GlobalConfig.service_mgr)
     end
   end
 

mgr/src/server/server.cr

@@ -82,7 +82,7 @@ module StorageMgr
     # Start all the services that were started previously
     services.each do |service|
       svc = Service.from_json(service.to_json)
-      svc.start
+      svc.start(plugin: GlobalConfig.service_mgr)
     end
   end
 
@@ -97,6 +97,7 @@ module StorageMgr
   def self.start(args)
     GlobalConfig.workdir = args.mgr_args.workdir
     GlobalConfig.logdir = args.mgr_args.logdir
+    GlobalConfig.service_mgr = args.mgr_args.service_mgr
 
     # Set the Datastore root directory
     Datastore.init(GlobalConfig.workdir)

mgr/src/server/services/plugins/helpers.cr

@@ -0,0 +1,7 @@
+class ServiceManagerException < Exception
+end
+
+abstract class ServiceManager
+  abstract def start(svc_id : String, cmd : Array(String))
+  abstract def stop(svc_id : String)
+end

mgr/src/server/services/plugins/systemd.cr

@@ -0,0 +1,63 @@
+require "../../plugins/helpers"
+require "./helpers"
+
+SYSTEMCTL     = "systemctl"
+UNIT_FILE_DIR = "/lib/systemd/system/"
+
+def unit_file_content(svc_id, cmd)
+  %Q[[Unit]
+Description=#{svc_id}
+After=network.target
+
+[Service]
+PIDFile=/run/kadalu/#{svc_id}.pid
+ExecStart=#{cmd}
+
+[Install]
+WantedBy=multi-user.target
+]
+end
+
+class SystemdServiceManager < ServiceManager
+  def initialize
+  end
+
+  def create(svc_id, cmd)
+    svc_file = "#{UNIT_FILE_DIR}/kadalu-#{svc_id}.service"
+    return if File.exists?(svc_file)
+
+    File.write(
+      svc_file,
+      unit_file_content(svc_id, cmd.join(" "))
+    )
+    rc, _out, err = execute(SYSTEMCTL, ["daemon-reload"])
+    if rc != 0
+      Log.warn &.emit("Systemd daemon-reload failed", svc_id: svc_id, err: err)
+    end
+  end
+
+  def delete(svc_id)
+    svc_file = "#{UNIT_FILE_DIR}/kadalu-#{svc_id}.service"
+    File.delete(svc_file) if File.exists?(svc_file)
+  end
+
+  def start(svc_id, cmd)
+    svc_id = svc_id.gsub("%2F", "-")
+    create(svc_id, cmd)
+    rc, _out, err = execute(SYSTEMCTL, ["start", "kadalu-#{svc_id}.service"])
+    raise ServiceManagerException.new(err) unless rc == 0
+  end
+
+  def stop(svc_id)
+    svc_id = svc_id.gsub("%2F", "-")
+    rc, _out, err = execute(SYSTEMCTL, ["stop", "kadalu-#{svc_id}.service"])
+    raise ServiceManagerException.new(err) unless rc == 0
+    delete(svc_id)
+  end
+
+  def is_running?(svc_id)
+    svc_id = svc_id.gsub("%2F", "-")
+    rc, _out, _err = execute(SYSTEMCTL, ["is-active", "--quiet", "kadalu-#{svc_id}.service"])
+    rc == 0
+  end
+end

mgr/src/server/services/services.cr

@@ -1,7 +1,10 @@
 require "json"
+require "log"
 
 require "moana_types"
 
+require "./plugins/*"
+
 module JSON::Serializable
   macro auto_json_discriminator(key)
     {% if @type.subclasses.size > 0 %}
@@ -14,6 +17,10 @@ module JSON::Serializable
     end
 end
 
+SERVICE_MGR_PLUGINS = {
+  "systemd" => SystemdServiceManager.new,
+}
+
 abstract class Service
   include JSON::Serializable
 
@@ -52,12 +59,19 @@ abstract class Service
     false
   end
 
-  def start
+  def start(plugin = "")
+    Log.debug &.emit("Starting the service", plugin: plugin, cmd: "#{path} #{args.join(" ")}")
     return if running?
 
     # Create PID file directory if not exists
     Dir.mkdir_p(Path[pid_file].parent)
 
+    mgr = SERVICE_MGR_PLUGINS[plugin]?
+    if mgr
+      mgr.start(id, [path] + args)
+      return
+    end
+
     @proc = Process.new(path, args)
     File.write(pid_file, "#{@proc.not_nil!.pid}") if @create_pid_file
     if @wait
@@ -69,7 +83,14 @@ abstract class Service
     end
   end
 
-  def stop(force = false)
+  def stop(plugin = "", force = false)
+    Log.debug &.emit("Stopping the service", plugin: plugin, cmd: "#{path} #{args.join(" ")}")
+    mgr = SERVICE_MGR_PLUGINS[plugin]?
+    if mgr
+      mgr.stop(id)
+      return
+    end
+
     if @proc.nil?
       begin
         pid = File.read(pid_file).strip.to_i
@@ -86,9 +107,9 @@ abstract class Service
     File.delete(pid_file) if File.exists?(pid_file)
   end
 
-  def restart(force = false)
-    stop(force)
-    start
+  def restart(plugin = "", force = false)
+    stop(plugin: plugin, force: force)
+    start(plugin)
   end
 
   def signal(sig)

tests/all/volumes.t

@@ -248,6 +248,7 @@ TEST "touch /mnt/vol15/d1/f{1..9}"
 TEST "touch /mnt/vol15/d2/f{1..9}"
 TEST "touch /mnt/vol15/d3/f{1..9}"
 TEST "kadalu volume expand DEV/vol15 server1:/exports/vol15/s1_e server2:/exports/vol15/s2_e server3:/exports/vol15/s3_e"
+TEST "sleep 3"
 
 EQUAL "3", (TEST "ls /exports/vol15/s1_e/* -d | wc -l").strip, "Check for fix-layout in server1 s1 unit"
 USE_NODE nodes[1]