manager

#!/bin/bash
#
# manager
#
# Starts and removes service instances based on CouchDb changes
#
set -u

function error
{
  echo "error: $1" >&2
  exit 1
}

function clean_up
{
  local service_name="$1"

  # Make sure there is only one instance running
  while read service
  do
    echo "[stopping ${service}]"
    systemctl stop "${service}"
  done < <(systemctl show \
             --type=service \
             --value \
             --state=running \
             --property=Id "$service_name"'@*' |\
             grep '.*@[[:digit:]]' | sort -rn -t '@' -k 2 | tail -n +2)
}

function ensure_started
{
  local service_name="$1"
  local rev="$2"

  if ! systemctl is-active -q "${service_name}@${rev}.service"
  then
    echo "[starting ${service_name}@${rev}]"
    systemctl start "${service_name}@${rev}.service" && \
        systemd-notify READY=1 STATUS="started instance ${rev}"
  fi
}

function stop
{
  local service_name="$1"
  local service_address="$2"

  trap '' EXIT TERM INT # ignore signals
  systemd-notify STOPPING=1 STATUS="Stopping instances"

  echo "[stopping all instances of ${service_name}@]"
  systemctl stop "${service_name}"'@[[:digit:]]*' # avoid stopping the manager

  echo "[removing ipvs service"]
  ipvsadm -D -t "$service_address"

  exit 0
}

function main
{
  local service_name="$1"
  local service_address="$2"
  local document_url="${COUCHDB_URL}/${service_name}"
  local feed="${COUCHDB_URL}/_changes?feed=continuous&heartbeat=true&filter=_doc_ids&doc_ids=\[\"$service_name\"\]"

  curl -Isf "$document_url" > /dev/null || error "Failed to fetch config document"

  # Setup ipvs
  echo "[adding ipvs virtual service $service_address]"
  modprobe ip_vs
  ipvsadm -A -t "$service_address" -s rr

  trap "stop $service_name $service_address" EXIT TERM INT

  while true
  do
    while read line
    do
      if [ ! -z "$line" ]
      then
        rev=$(jq -r '.changes[0] | .rev // ""' <<< "$line")
        [ -z "$rev" ] || ensure_started "$service_name" "$rev"
      fi
      clean_up "$service_name"
    done < <(curl -Ns "$feed")

    echo "warning: lost connection to CouchDb"
    sleep 5
  done
}

(( $# != 2 )) && error "usage: $0 service-name service-address"
main "$1" "$2"