#!/bin/bash
#FIXME Could not check service availability by exit states because of RCM-436, had to switch to PID checking
#TODO Check if $WF_USER user exists

# Remember last non-zero exit state (helps with tee piping)
set -o pipefail

# In case localized output is used and we need to parse it
export LANG="en_US.UTF-8"

# Check for necessary utils and assign corresponding variables
for i in tty ps grep tee whoami basename dirname date mkdir mktemp mailx rsync ssh tar; do
  which $i >/dev/null 2>&1 && declare "$(echo $i | sed 's/-/_/g' | tr [:lower:] [:upper:])"="$(which --skip-alias $i)" || { echo "$i utility was not found!"; exit 1; }
done

THIS="cold-backup"
THIS_DIR="$($DIRNAME $0)"
CONF_DIR="$THIS_DIR"
TMPLOG=$($MKTEMP -u)

HOST="$HOSTNAME"
WF_USER="wfuser"
RUN_AS_ROOT="false"
SSH_HOST="example.com"
SSH_PORT="22"
SSH_ARGS="-q -o UserKnownHostsFile=/dev/null -o ConnectTimeout=1 -o ConnectionAttempts=1 -o StrictHostKeyChecking=no -o BatchMode=yes"
RSYNC_ARGS="-azogh --delete --no-whole-file --progress"
MAILFROM="root@$HOST"
MAILTO="root"
NOTIFY_ALWAYS="false"
DATE_FORMAT="%F %T %Z"
LOG_DIR="/opt/workfusion/log"
MYSQL_DIR="/opt/workfusion/apps/mysql"
PGSQL_DIR="/opt/workfusion/apps/pgsql/9.4/data"
S3EMU_DIR="/opt/workfusion/apps/s3emu"
TOMCAT_DIR="/opt/workfusion/apps/webapps/tomcat"
MYSQL_DATA_DIR="$MYSQL_DIR/data"
PGSQL_DATA_DIR="$PGSQL_DIR"
S3EMU_DATA_DIR="$S3EMU_DIR/data"
MYSQL_SVC="MySQL:$MYSQL_DIR/mysql.service:$MYSQL_DIR/mysql.sock"
PGSQL_SVC="PostgreSQL:$PGSQL_DIR/postgresql-9.4.service:$PGSQL_DIR/run/postgresql-9.4.init.pid"
RIAK_SVC="Riak:$S3EMU_DIR/riak.service:$S3EMU_DIR/data/riak/run/riak/riak.pid"
RIAKCS_SVC="Riak-CS:$S3EMU_DIR/riak-cs.service:$S3EMU_DIR/data/riak-cs/run/riak-cs/riak-cs.pid:10"
STANCHION_SVC="Stanchion:$S3EMU_DIR/stanchion.service:$S3EMU_DIR/data/stanchion/run/stanchion/stanchion.pid:10"
STOP_ORDER="$MYSQL_SVC $PGSQL_SVC $RIAK_SVC $RIAKCS_SVC $STANCHION_SVC"
START_ORDER="$MYSQL_SVC $PGSQL_SVC $RIAK_SVC $RIAKCS_SVC $STANCHION_SVC"
TOMCAT_CTRL="/etc/init.d/tomcat"
BACKUP_DIR="/opt/workfusion/backup/cold-backup"
BACKUP_TMP_DIR="$BACKUP_DIR/.tmp"
STOP_TIMEOUT=30
START_TIMEOUT=30

out() {
  if [ "$2" == "e" ]; then STATUS="ERROR"; C=91; elif [ "$2" == "w" ]; then STATUS="WARN"; C=93; else STATUS="INFO"; C=92; fi
  # Output in color only if script is running in interactive mode
  $TTY >/dev/null && echo -e "\033[0;96m$THIS: \033[0;${C}m$1\033[0m"
  echo -e "$($DATE +"[$DATE_FORMAT]") $STATUS: $1" >> $TMPLOG
}

finish() {
  [ -z $1 ] && STATE=$? || STATE=$1
  # Excluding EXIT to avoid multiple triggering
  trap - EXIT
  if [ -f $TMPLOG ]; then
    if [ "$NOTIFY_ALWAYS" == "true" ] || ! $TTY >/dev/null && [ $STATE -ne 0 ]; then
      $GREP ERROR $TMPLOG >/dev/null && TITLE="ERROR" || TITLE="INFO"
      # Using tr to make the output readable in Outlook https://access.redhat.com/solutions/1136493
      cat $TMPLOG | tr -d "\015" | $MAILX -s "$TITLE: $HOST $THIS" -r $MAILFROM $MAILTO
    fi
    cat $TMPLOG >> $LOG_FILE
    rm $TMPLOG
  fi
  exit $STATE
}

start_services() {
  out "Starting services..."
  for i in $START_ORDER; do
    NAME="$(echo $i | cut -d: -f1)"; CTRL="$(echo $i | cut -d: -f2)"; PID="$(echo $i | cut -d: -f3)"; NAP="$(echo $i | cut -d: -f4)"
    if [ ! -e "$PID" ]; then
      # Riak refused to start properly unless pauses are made
      [ "$NAP" != "" ] && { out "Having a $NAP second delay before $NAME startup..."; sleep $NAP; }
      [ $RUN_AS_ROOT == "true" ] && su $WF_USER -lc "$CTRL start" >> $TMPLOG 2>&1 || $CTRL start >> $TMPLOG 2>&1
      i=0; while [ ! -e "$PID" ]; do
        [ $i -eq $START_TIMEOUT ] && { out "Timeout reached while starting $NAME service!" e; HOOK=1; break; } || ((++i))
        sleep 1; echo "Waiting for $NAME startup for $i sec..."
      done
      [ -e "$PID" ] && out "$NAME service was started successfully."
    else
      out "$NAME service is already running."
    fi
  done

  # Start Tomcat
  if [ ! -z "$REMOTE" ]; then
    if $REMOTE "$PS aux | $GREP $TOMCAT_DIR/conf | $GREP -v grep" >/dev/null 2>&1; then
      out "Tomcat service is already running."
    else
      $REMOTE "$TOMCAT_CTRL start" >> $TMPLOG 2>&1
      i=0; while ! $REMOTE "$PS aux | $GREP $TOMCAT_DIR/conf | $GREP -v grep" >/dev/null 2>&1; do
        [ $i -eq $START_TIMEOUT ] && { out "Timeout reached while starting Tomcat service!" e; break; } || ((++i))
        sleep 1; echo "Waiting for Tomcat startup for $i sec..."
      done
      [ -e "$PID" ] && out "Tomcat service was started successfully."
    fi
  fi
  [ -z $HOOK ] && return 0 || return 1
}

LOG_FILE="$LOG_DIR/$THIS.log"

RUN_AS_ROOT="$(echo $RUN_AS_ROOT | tr [:upper:] [:lower:])"
NOTIFY_ALWAYS="$(echo $NOTIFY_ALWAYS | tr [:upper:] [:lower:])"

# Pre-flight checks
[ -f "$CONF_DIR/$THIS.conf" ] && . "$CONF_DIR/$THIS.conf" || { out "Config file not found! Aborting!" e; finish 1; }
if [ "$RUN_AS_ROOT" == "false" -a "$($WHOAMI)" != "$WF_USER" ]; then
  out "This script must be run as $WF_USER! Aborting!" e; finish 1
elif [ "$RUN_AS_ROOT" != "false" -a "$($WHOAMI)" != "root" ]; then
  out "This script must be run as root! Aborting!" e; finish 1
fi
[ $($PS h -C "$THIS" | $GREP -wcv $$) -gt 1 ] && { out "Another instance of $THIS is running! Aborting!" e; finish 1; }

[ ! -d "$MYSQL_DATA_DIR" ] && { out "MySQL data directory was not found! Aborting!" e; finish 1; }
[ ! -r "$MYSQL_DATA_DIR" ] && { out "MySQL data directory is not readable! Aborting!" e; finish 1; }
[ ! -d "$PGSQL_DATA_DIR" ] && { out "PostgreSQL data directory was not found! Aborting!" e; finish 1; }
[ ! -r "$PGSQL_DATA_DIR" ] && { out "PostgreSQL data directory is not readable! Aborting!" e; finish 1; }
[ ! -d "$S3EMU_DATA_DIR" ] && { out "S3 Emulator data directory was not found! Aborting!" e; finish 1; }
[ ! -r "$S3EMU_DATA_DIR" ] && { out "S3 Emulator data directory is not readable! Aborting!" e; finish 1; }

# SSH connection check
if [ "$SSH_HOST" != "" ]; then
  out "Testing connection to $SSH_HOST..."
  REMOTE="$SSH $SSH_ARGS -p $SSH_PORT $WF_USER@$SSH_HOST"
  $REMOTE "exit" && out "Connection successful." || { out "Cannot connect to $SSH_HOST! Aborting!" e; finish 1; }
else
  out "Skipping Tomcat shutdown as SSH_HOST variable is not set in $THIS.conf!" w
fi

# Creating directories before actual run
for i in "$LOG_DIR" "$BACKUP_DIR" "$BACKUP_TMP_DIR"{mysql,pgsql,s3emu}; do
  $MKDIR -p "$i" >/dev/null 2>&1 || { out "Could not create $i directory as it is not writable! Aborting!" e; finish 1; }
done

# Trapping signals
trap '[ $? -gt 0 ] && out "Error while running $THIS!" e; start_services; finish 1' TERM INT HUP KILL EXIT

out "Stopping services..."

# Stopping Tomcat
if [ ! -z "$REMOTE" ]; then
  if $REMOTE "$PS aux | $GREP $TOMCAT_DIR/conf | $GREP -v grep" >/dev/null 2>&1; then
    $REMOTE "$TOMCAT_CTRL stop" >> $TMPLOG 2>&1
    i=0; while $REMOTE "$PS aux | $GREP $TOMCAT_DIR/conf | $GREP -v grep" >/dev/null 2>&1; do
      [ $i -eq $STOP_TIMEOUT ] && { out "Timeout reached while stopping Tomcat service! Aborting!" e; exit 1; } || ((++i))
      sleep 1; echo "Waiting for Tomcat shutdown for $i sec..."
    done
    out "Tomcat service was stopped successfully."
  else
    out "Tomcat service is not running!" w
  fi
fi

# Stopping other services
for i in $STOP_ORDER; do
  NAME="$(echo $i | cut -d: -f1)"; CTRL="$(echo $i | cut -d: -f2)"; PID="$(echo $i | cut -d: -f3)"
  if [ -e "$PID" ]; then
    [ $RUN_AS_ROOT == "true" ] && su $WF_USER -lc "$CTRL stop" >> $TMPLOG 2>&1 || $CTRL stop >> $TMPLOG 2>&1
    i=0; while [ -e "$PID" ]; do
      [ $i -eq $STOP_TIMEOUT ] && { out "Timeout reached while stopping $NAME service! Aborting!" e; exit 1; } || ((++i))
      sleep 1; echo "Waiting for $NAME shutdown for $i sec..."
    done
    out "$NAME service was stopped successfully."
  else
    out "$NAME service is not running!" w
  fi
done

out "Performing MySQL backup..."
$RSYNC $RSYNC_ARGS $MYSQL_DATA_DIR/ $BACKUP_TMP_DIR/mysql |& $TEE -a $TMPLOG 2>&1 && out "MySQL backup was successful." || { out "There was error while trying to backup MySQL! Aborting!" e; exit 1; }
out "Performing PostgreSQL backup..."
$RSYNC $RSYNC_ARGS $PGSQL_DATA_DIR/ $BACKUP_TMP_DIR/pgsql |& $TEE -a $TMPLOG 2>&1 && out "PostgreSQL backup was successful." || { out "There was error while trying to backup PostgreSQL! Aborting!" e; exit 1; }
out "Performing S3 Emulator backup..."
$RSYNC $RSYNC_ARGS $S3EMU_DATA_DIR/ $BACKUP_TMP_DIR/s3emu |& $TEE -a $TMPLOG 2>&1 && out "S3 Emulator backup was successful." || { out "There was error while trying to backup S3 Emulator! Aborting!" e; exit 1; }

start_services

for i in mysql pgsql s3emu; do
  out "Creating 'backup-${i}.tar.gz'..."
  [ -e "$BACKUP_DIR/backup-${i}.tar.gz" ] && { out "Moving existing backup to backup-${i}-old.tar.gz"; mv -f "$BACKUP_DIR/backup-${i}.tar.gz" "$BACKUP_DIR/backup-${i}-old.tar.gz"; }
  if $TAR -cvzf "$BACKUP_DIR/backup-${i}.tar.gz" -C $BACKUP_TMP_DIR/$i . |& $TEE -a $TMPLOG 2>&1; then
    rm -f "$BACKUP_DIR/backup-${i}-old.tar.gz"
    out "Successfully created 'backup-${i}.tar.gz'."
  else
    mv -f "$BACKUP_DIR/backup-${i}-old.tar.gz" "$BACKUP_DIR/backup-${i}.tar.gz"
    out "There was error while creating 'backup-${i}.tar.gz'!" e
  fi
done

out "All done!"

exit 0
