rtcds.in

#!/bin/bash -e

log() {
    echo "$@" >&2
}

RTS_VERSION=${RTS_VERSION:-__VERSION__}
if [[ "$RTS_VERSION" =~ ._VERSION_. ]] ; then
    log "RTS_VERSION variable not set."
    exit 1
fi

# this should define all USER_VARS (see below)
ENV_FILE=${RTS_ENV:-/etc/advligorts/env}
HOST_ENV_FILE=/etc/advligorts/systemd_env_`hostname`
set -o allexport
source "$ENV_FILE" 2>/dev/null || true
if [[ -e $HOST_ENV_FILE ]] ; then
    log "found host specific environment file '$HOST_ENV_FILE'"
    source $HOST_ENV_FILE 2>/dev/null || true
fi
set +o allexport

SITE=${SITE^^*}
site=${SITE,,*}
IFO=${IFO^^*}
ifo=${IFO,,*}

CHECK_MODEL_IN_HOST=${CHECK_MODEL_IN_HOST:-false}
IS_DOLPHIN_NODE=${IS_DOLPHIN_NODE:-false}
DAQ_STREAMING=${DAQ_STREAMING:-false}

RCG_SRC=${RCG_SRC:-/usr/share/advligorts/src}
RCG_BUILD_ROOT=${RCG_BUILD_ROOT:-/var/cache/advligorts}
RCG_BUILDD=${RCG_BUILDD:-$RCG_BUILD_ROOT/rcg-$RTS_VERSION}
# FIXME: the RCG hard-codes /opt/rtcds as the root
RCG_TARGET=/opt/rtcds/${site}/${ifo}

# search paths for C source code
CDS_SRC=${CDS_SRC:-$RCG_LIB_PATH}
CDS_IFO_SRC=${CDS_IFO_SRC:-$CDS_SRC}

# add RCG module source to lib path
# FIXME: rename RCG_MOD_PATH
RCG_LIB_PATH=$RCG_LIB_PATH:${RCG_SRC}/src/epics/simLink/:${RCG_SRC}/src/epics/simLink/lib

#options from environment variables
ALLOW_MODEL_ENABLE=${ALLOW_MODEL_ENABLE:-true}

USER_VARS=(SITE IFO RCG_LIB_PATH)
LIST_VARS=(RTS_VERSION ${USER_VARS[@]} RCG_SRC RCG_BUILD_ROOT RCG_BUILDD RCG_TARGET)
EXPORT_VARS=(${USER_VARS[@]} site ifo CDS_SRC CDS_IFO_SRC)

# time in seconds to wait before starting another model.
START_DELAY=${START_DELAY:-15}

# set the umask to ensure that all files and directories are made
# group writable
umask 0002

USE_KERNEL_MODELS=${USE_KERNEL_MODELS:-true}

# for all flags, empty string means cleared.  Any other string means set.
if $USE_KERNEL_MODELS ; then
    log "Build kernel-mode models by default"
    USERLAND_FLAG=
    NOKERNELSPACE_FLAG=
else
    log "Build user mode models by default"
    USERLAND_FLAG=0
    NOKERNELSPACE_FLAG=0
fi

ALL_FLAG=
FOLLOW_FLAG=
INFO_FLAG=
VERSION_FLAG=
HELP_FLAG=
FORCE_START_FLAG=

# capture all command line flags as variables
params=""
ccmd=""
while (( "$#" )); do
    case "$1" in
        --all)
            ALL_FLAG=0
            shift
            ;;
        --user-space|--userland)
            USERLAND_FLAG=0
            shift
            ;;
        --no-user-space|--no-userland)
            USERLAND_FLAG=''
            shift
            ;;
        --kernel-space)
            NOKERNELSPACE_FLAG=''
            shift
            ;;
        --no-kernel-space)
            NOKERNELSPACE_FLAG=0
            shift
            ;;
        -f)
            FOLLOW_FLAG=0
            shift
            ;;
        -i)
            INFO_FLAG=0
            shift
            ;;
        version|-v|--version)
            VERSION_FLAG=0
            shift
            ;;
        help|-h|--help)
            HELP_FLAG=0
            shift
            ;;
        --force-start)
            FORCE_START_FLAG=0
            shift
            ;;
        --*|-*)  # unknown flags
            log "Error: Unknown flag $1"
            exit 1
            ;;
        *)  #handle non-flags
            params="${params} $1"
            shift
            ;;
    esac
done

eval set -- "$params"

##################################################

contains() {
    local x=$1
    shift
    [[ $@ =~ (^|[[:space:]])$x($|[[:space:]]) ]] && return 0 || return 1
}

epics_only_model() {
    contains $1 $EPICS_ONLY_MODELS && return 0 || return 1
}

# echo the systemd unit that should be used to control the model
get_model_unit() {
    local sys=$1
    if epics_only_model ${sys} ; then
	echo "rts-epics@${sys}"
    else
	echo "rts@${sys}.target"
    fi
}

check_env() {
    for var in ${USER_VARS[*]} ; do
	if [ ! "${!var}" ] ; then
	    log "ERROR: variable '$var' not set."
	    log "The following environment variables must be set (in e.g. $ENV_FILE):"
	    for vv in ${USER_VARS[*]} ; do
		log "   $vv"
	    done
	    exit 1
	fi
    done
    for var in ${LIST_VARS[*]} ; do
	log "$var=${!var}"
    done
    for var in ${EXPORT_VARS[*]} ; do
	export $var
    done
}

prep_buildd() {
    if [ -d "$RCG_BUILDD" ] ; then
        return
    fi
    log "creating RCG_BUILDD $RCG_BUILDD..."
    if ! mkdir -p "$RCG_BUILDD" 2>/dev/null ; then
        log "Could not create build directory '$RCG_BUILDD'."
        log "Please create manually (with correct permissions) and try again."
        exit 1
    fi
    log "configuring RCG_BUILDD $RCG_BUILDD..."
    cd "$RCG_BUILDD"
    "$RCG_SRC"/configure
}

prep_target(){
    if [ -d "$RCG_TARGET" ] ; then
        return
    fi
    log "creating RCG_TARGET $RCG_TARGET..."
    if ! mkdir -p "$RCG_TARGET" 2>/dev/null ; then
        log "Could not create target directory '$RCG_TARGET'."
        log "Please create manually (with correct permissions) and try again."
        exit 1
    fi
    mkdir -p ${RCG_TARGET}/{target,chans/tmp}
}

########

_systemctl_list_units() {
    systemctl list-units --all --full --plain --no-legend --no-pager rts@*.target \
	| awk '{print $1}' \
	| sed 's/^rts@\([a-z0-9]*\)\.target$/\1/'
}

list_host_sys_systemd() {
    _systemctl_list_units | grep 'iop' || true
    _systemctl_list_units | grep -v 'iop' || true
}

list_host_sys() {
  if [[ -e /etc/rtsystab ]] && [[ -e /etc/rt_fe.sh ]] ; then
    /etc/rt_fe.sh | sed 's/ /\n/g'
  else
    list_host_sys_systemd
  fi
}

# returns 0 if $1 names a system that's designated to run
#  on this host
check_sys_in_host() {
    if $CHECK_MODEL_IN_HOST && [[ -z $FORCE_START_FLAG ]] ; then
  	local systems=`list_host_sys`
  	contains $1 $systems
    fi
}

check_unit_available() {
  if ! systemctl cat rts@$1.target > /dev/null ; then
    log "RTS front end systemd units don't seem to be available.  Is the advligorts-fe package installed?"
    exit 5
  fi
}

check_host_sys_enabled() {
    for sys ; do
      check_unit_available ${sys}
	if ! systemctl is-enabled rts@${sys}.target > /dev/null ; then
	    log "System '${sys}' is not enabled on this system, first enable with 'enable ${sys}'."
	    exit 4
	fi
    done
}

check_host_sys_active() {
    for sys ; do
        check_unit_available ${sys}
        local unit_name=`get_model_unit ${sys}`
	if ! systemctl is-active $unit_name > /dev/null ; then
	    log "System '${sys}' is not active on this system'."
	    exit 4
	fi
    done
}

check_host_sys_inactive() {
    for sys ; do
        check_unit_available ${sys}
        local unit_name=`get_model_unit ${sys}`
	if systemctl is-active $unit_name > /dev/null ; then
	    log "System '$sys' is already active on this system'."
	    exit 4
	fi
    done
}

########

setup_make_args() {
    make_args=''
    if [[ -n $USERLAND_FLAG ]] ; then
        make_args='RCG_BUILD_USP=YES'
    fi
    if [[ -n $NOKERNELSPACE_FLAG ]] ; then
        make_args="$make_args RCG_BUILD_NO_KOBJ=YES"
    fi
}

build_sys() {
    cd $RCG_BUILDD
    for sys ; do
	log "### building $sys..."
    setup_make_args
	make $sys $make_args
    done
}

build_world() {
    cd $RCG_BUILDD
    log "### building world..."
    setup_make_args
    make -i World $make_args
}

install_sys() {
    cd $RCG_BUILDD
    for sys ; do
	log "### installing $sys..."
	make install-$sys
    done
}

install_world() {
    cd $RCG_BUILDD
    log "### installing world..."
    make installWorld
}

start_sys() {
    for sys ; do
	log "### starting $sys..."
	if check_sys_in_host $sys ; then
	  local unit_name=`get_model_unit ${sys}`
	  sudo systemctl start $unit_name

	  # pause needed to avoid timing glitch when user model started immediately after IOP
	  log "... waiting for start of ${sys} to complete"
	  sleep $START_DELAY
	  log "${sys} started"
	else
	  log "$sys can't be run on $(hostname)"
	  log "Override with the '--force-start' option."
	fi
    done
}

stop_sys() {
    for sys ; do
	log "### stopping $sys..."
	#${SCRIPTD}/kill${sys}
	local unit_name=`get_model_unit ${sys}`
	sudo systemctl stop $unit_name

	md=waiting
	while [ -n "${md}" ]
	do
	  sleep 1
	  md=$(lsmod | grep "^${sys}\s") || true
	done
    done
}

enable_sys() {
    for sys ; do
	log "### enabling $sys..."
	sudo systemctl enable rts@${sys}.target
    done
}

disable_sys() {
    for sys ; do
	log "### disabling $sys..."
	sudo systemctl disable rts@${sys}.target
    done
}

_lsmod() {
    local systems=($(list_host_sys))
    local rts=(mbuf gpstime)
    local dolphin=(dis_kosif dis_ix_ntb dis_ix_dma dis_irm dis_sisci dis_intel_dma)

    # FIXME: how to test for needed modules?
    if $IS_DOLPHIN_NODE ; then
    	local modules=(${rts[*]} ${dolphin[*]})
    else
    	local modules=(${rts[*]})
    fi
    
    local allloaded=
    for m in ${modules[*]}; do
	md=$(lsmod | grep "^${m}\s") || true
	if [ -z "$md" ] ; then
	    printf "%-18s ***NOT LOADED***\n" "$m"
	    allloaded=1
	else
	    echo "$md"
	fi
    done
    if [ "$systems" ] ; then
	echo
	for m in ${systems[*]}; do
	    if epics_only_model $m ; then
		test 0
	    else
		md=$(lsmod | grep "^${m}\s") || true
		if [ -z "$md" ] ; then
		    printf "%-18s ***NOT LOADED***\n" "$m"
		    allloaded=1
		else
		    echo "$md"
		fi
	    fi
	done
    fi
    if [ "$allloaded" ] ; then
	return 4
    fi
}

systems_status() {
  local systems=$(list_host_sys)
  local subsystems=(epics module awgtpman)
  local systems_health=0

  echo "System Status"

  local header="system"
  printf '%-15s' system
  for subsys in ${subsystems[*]} ; do
    printf '%-15s' ${subsys}
  done
  echo
  echo "------------------------------------------------------"

  for sys in ${systems[*]} ; do
    printf '%-15s ' $sys
    for subsys in ${subsystems[*]} ; do
      local subsysstatus='OFF'
      if ! epics_only_model $sys || [[ $subsys == 'epics' ]] ; then
          if systemctl is-failed "rts-${subsys}@${sys}" > /dev/null ; then
            subsysstatus='FAILED'
            systems_health=10
          else
            if systemctl is-active "rts-${subsys}@${sys}" > /dev/null ; then
              subsysstatus='ON'
            else
              systems_health=10
            fi
          fi
      else
          subsysstatus='N/A'
      fi
      printf '%-15s' $subsysstatus
    done
    echo
  done
  return $systems_health
}

streamer_status() {
  local services=(rts-local_dc rts-transport@cps_xmit)
  local streamer_health=0

  echo "Streamer Status"
  for service in ${services[*]} ; do
    local health="OFF"
    if systemctl is-failed $service > /dev/null ; then
      health="FAILED"
      streamer_health=11
    else
      if systemctl is-active $service > /dev/null ; then
        health="ON"
      else
        streamer_health=11
      fi
    fi
    printf '%-25s%-15s\n' $service $health
  done

  return $streamer_health
}

global_status() {
  echo "Kernel Module Status\n"


  echo
  local module_health=0
  if _lsmod ; then
    echo "Kernel Module Status = ALL LOADED"
  else
    module_health=12
    echo "Kernel Module Status = SOME MISSING"
  fi

  echo
  local systems_health=0
  echo
  if systems_status ; then
    echo "Systems Status =       ALL ACTIVE"
  else
    echo "Systems Status =       DEGRADED"
    systems_health=12
  fi

  echo

  local streamer_health=0
  if $DAQ_STREAMING ; then
    echo
    if streamer_status ; then
      echo "Streamer Status =      ALL ACTIVE"
    else
      echo "Streamer Status =      DEGRADED"
      streamer_health=12
    fi
  fi

  echo
  if [[ $systems_health == 0 ]] && [[ $streamer_health == 0 ]] && [[ $module_health == 0 ]] ; then
    echo "Overall Status =       OK"
    return 0
  else
    echo "Overall Status =       DEGRADED"
    return 12
  fi
}

############################################################

usage() {

  if $USE_KERNEL_MODELS ; then
      local udefault=""
      local kdefault=" (default)"
  else
      local udefault=" (default)"
      local kdefault=""
  fi

  if $ALLOW_MODEL_ENABLE ; then
    local enable_usage="
  enable <sys>...|--all            enable system start at boot"
  else
    local enable_usage="
"
  fi

    echo "Usage: $(basename $0) <command> [options] [args]

Advanced LIGO Real Time System control interface.

Available commands:

  build|make <sys>...|--all         build system
    --user-space|--userland         build model in userland $udefault
    --no-user-space|--no-userland   do not build model in userland $kdefault
    --kernel-space                  build model in kernel space $kdefault
    --no-kernel-space               do not build model in kernel space $udefault
  install <sys>...|--all            install system

  list|ls                           list systems for host
  ${enable_usage}
  disable <sys>...|--all            disable system start at boot
  start <sys>...|--all              start systems
    --force-start                   start a model even if it fails some checks
  restart <sys>...|--all            restart running systems
  stop|kill <sys>...|--all          stop running systems

  status [<sys>]                    show status for system services

  log                               show logs for system services
    -f                              follow logs
  blog <sys>                        show last build log for system
    -i                              print log paths
  lsmod                             list loaded RTS kernel modules
  env                               print system environment info
  version|--version|-v              print version
  help|--help|-h                    this help
"
}

if [[ -n $HELP_FLAG ]]; then
    usage
    exit 0
fi

if [[ -n $VERSION_FLAG ]] ; then
    echo $RTS_VERSION
    exit 0
fi

if [[ $1 ]] ; then
    cmd=$1
    shift
else
    log "You must specify a command."
    log
    usage
    exit 1
fi


case $cmd in
    'build'|'make')
    	if [ "$EUID" -eq 0 ] ; then
    	    log "*** Error: building as 'root' not allowed"
    	    exit 2
    	fi
	if [[ -n $ALL_FLAG ]] ; then
	    check_env
	    prep_buildd
	    build_world
	elif [[ $1 ]] ; then
	    check_env
	    prep_buildd
	    build_sys $@
	else
	    log "You must specify at least one system to build."
	    exit 2
	fi
	;;
    'install')
        if [ "$EUID" -eq 0 ] ; then
	    log "*** Error: Installing as 'root' not allowed"
	    exit 2
	fi
	if [[ -n $ALL_FLAG ]] ; then
	    check_env
	    prep_target
	    install_world
	elif [[ $1 ]] ; then
	    check_env
	    prep_target
	    install_sys $@
	else
	    log "You must specify at least one system to install."
	    exit 2
	fi
	;;
    'start')
	if [[ -n $ALL_FLAG ]] ; then
	    start_sys $(list_host_sys)
	elif [[ $1 ]] ; then
	    start_sys $@
	else
	    log "You must specify at least one system to start (or '--all')."
	    exit 2
	fi
	;;
    'restart')
	if [[ -n $ALL_FLAG ]] ; then
	    # we do this in reverse so the IOP is stopped last
	    stop_sys $(list_host_sys | tac)
	    start_sys $(list_host_sys)
	elif [[ $1 ]] ; then
	    stop_sys $@
	    start_sys $@
	else
	    log "You must specify at least one system to restart (or '--all')."
	    exit 2
	fi
	;;
    'stop'|'kill')
	if [[ -n $ALL_FLAG ]] ; then
	    # we do this in reverse so the IOP is stopped last
	    stop_sys $(list_host_sys | tac)
	elif [[ $1 ]] ; then
	    stop_sys $@
	else
	    log "You must specify at least one system to stop (or '--all')."
	    exit 2
	fi
	;;
    'enable'|'disable')
	if [[ ${cmd} == 'enable' ]] && ! $ALLOW_MODEL_ENABLE ]] ; then
	    log "Enabling models is not allowed on this host."
	    log "ALLOW_MODEL_ENABLE environment variable is set to false."
	    exit 2
	fi
	if [[ -n $ALL_FLAG ]] ; then
	    # we do this in reverse so the IOP is stopped last
	    ${cmd}_sys $(list_host_sys | tac)
	elif [[ $1 ]] ; then
	    ${cmd}_sys $@
	else
	    log "You must specify at least one system to $cmd (or '--all')."
	    exit 2
	fi
	;;
    'status')
	if [ -z "$1" ] || [[ -n $ALL_FLAG ]] ; then
	    global_status
	else
	    sys=$1
	    systemctl status rts@${sys}.target rts-{awgtpman,epics,module}@${sys}.service
	fi
    ;;
    'log')
	args=
	if [[ -n $FOLLOW_FLAG ]] ; then
	    args="-f"
	fi
	if [ -z "$1" ] ; then
	    log "You must specify one system to show log."
	    exit 2
	fi
	sys=$1
	sudo journalctl $args --unit=rts*${sys}*
	;;
    'blog')
	format=full
	if [[ -n $INFO_FLAG ]] ; then
	    format=info
	fi
	if [ -z "$1" ] ; then
	    log "You must specify system to view."
	    exit 2
	fi
	olog="${RCG_BUILDD}/${1}.log"
	elog="${RCG_BUILDD}/${1}_error.log"
	echo $olog
	case "$format" in
	    info)
		ls -al "$olog" "$elog"
		;;
	    full)
		sed 's/^/stdout:/' < "$olog"
		sed 's/^/stderr:/' < "$elog"
		;;
	esac
	;;
    'list'|'ls')
	list_host_sys
	;;
    'lsmod'|'lsmods')
	_lsmod
	;;
    'env')
	check_env
	;;
    *)
	log "Unknown command: $cmd"
	log
	usage
	exit 1
	;;
esac