#!/usr/bin/env bash
# $Id$
#
# Thomas Dreibholz's PlanetLab Script Collection
# Copyright (C) 2005-2023 by Thomas Dreibholz
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
# Contact: network@iem.uni-due.de


if [ -e ./servicecontrol ] ; then
   SERVICE_CONTROL_CMD=./servicecontrol
elif [ -e ../../servicecontrol ] ; then
   SERVICE_CONTROL_CMD=../../servicecontrol
else
   echo >&2 "ERROR: servicecontrol not found!"
   exit 1
fi
if [ -e ./ssh-tool ] ; then
   SSH_TOOL_CMD=./ssh-tool
elif [ -e ../../ssh-tool ] ; then
   SSH_TOOL_CMD=../../ssh-tool
else
   echo >&2 "ERROR: ssh-tool not found!"
   exit 1
fi

# ====== Set defaults =======================================================
if [ "$1" = "init" ] ; then
   export Measurement=tos-pr-failureI
   export Duration=120
   export PRs=1
   export PEs=10
   export PUs=30
   export PR1UptimeOverride="auto"
   export PEAllUptime=0
   export PEAllDowntime=0
   export PRAllUptime=0
   export PRAllDowntime=0
   export UsePEMasterPR="false"
   export UsePUMasterPR="false"
   export AttackInterval=1
   export Policy="LeastUsed"
   export LoadDegOverride=0
   export UseTakeoverSuggestion="false"
   export MaxBadPEReports=3
   export EKATransmissionInterval=5000
   export EKATimeoutInterval=1000
   export MaxHRRate=-1
   export MaxEURate=-1
   export Capacity=1000000
   export MaxJobs=10
   export ReregInterval=30000
   export SKATransmissionInterval=2500
   export SKATimeoutInterval=2500
   export CookieMaxCalculations=5000000
   export CookieMaxTime=604800
   export CleanShutdownProbability=0.0
   export JobInterval=60.0
   export JobSize=10000000
   export Attackers=0
   export AttackType="registration"
   export AttackIdentifier=0
   export AttackReportUnreachableProbability=0.0

# ====== Run scenario =======================================================
else

   check_variable ()
   {
      if [ "$1" = "" ] ; then
         echo >&2 "ERROR: Variable $1 is not set!"
         exit 1
      fi
      echo "   $1=$2"
   }


#    echo "==== TEST ONLY ==== !!!!!!!!!!!!!!!!!!!!"
#    SSH_USER="dreibh"
#    SSH_KEY="/home/dreibh/.ssh/test-key"
#    SERVICE_DIRECTORY="src/rsplib2/rsplib"
#    SERVICE_PRE_PROGRAM=""
#    OPT_CSP="-cspserver=127.0.0.1:2960 -cspinterval=1000"
#
#    PRNodeList="localhost"
#    PENodeList="localhost"
#    PUNodeList="localhost"
#    AttackerNodeList="localhost"
#
#    . ./$0 init
#    PRs=5
#    Duration=10
#    echo "==== TEST ONLY ==== !!!!!!!!!!!!!!!!!!!!"


   check_variable SSH_USER $SSH_USER
   check_variable SSH_KEY $SSH_KEY
   check_variable SERVICE_DIRECTORY $SERVICE_DIRECTORY
   check_variable SERVICE_PRE_PROGRAM $SERVICE_PRE_PROGRAM

   check_variable PRNodeList $PRNodeList
   check_variable PENodeList $PENodeList
   check_variable PUNodeList $PUNodeList
   check_variable AttackerNodeList $AttackerNodeList

   check_variable PortBase

   check_variable Measurement $Measurement
   check_variable Duration $Duration
   check_variable PRs $PRs
   check_variable PEs $PEs
   check_variable PUs $PUs
   check_variable PEAllUptime $PEAllUptime
   check_variable PEAllDowntime $PEAllDowntime
   check_variable PR1UptimeOverride $PR1UptimeOverride
   check_variable PRAllUptime $PRAllUptime
   check_variable PRAllDowntime $PRAllDowntime
   check_variable UsePEMasterPR $UsePEMasterPR
   check_variable UsePUMasterPR $UsePUMasterPR
   check_variable LoadDegOverride $LoadDegOverride
   check_variable Policy $Policy
   check_variable UseTakeoverSuggestion $UseTakeoverSuggestion
   check_variable MaxBadPEReports $MaxBadPEReports
   check_variable EKATransmissionInterval $EKATransmissionInterval
   check_variable EKATimeoutInterval $EKATimeoutInterval
   check_variable Capacity $Capacity
   check_variable MaxJobs $MaxJobs
   check_variable ReregInterval $ReregInterval
   check_variable SKATransmissionInterval $SKATransmissionInterval
   check_variable SKATimeoutInterval $SKATimeoutInterval
   check_variable CookieMaxCalculations $CookieMaxCalculations
   check_variable CookieMaxTime $CookieMaxTime
   check_variable CleanShutdownProbability $CleanShutdownProbability
   check_variable JobInterval $JobInterval
   check_variable JobSize $JobSize
   check_variable Attackers $Attackers
   check_variable AttackType $AttackType
   check_variable AttackInterval $AttackInterval
   check_variable AttackIdentifier $AttackIdentifier
   check_variable AttackReportUnreachableProbability $AttackReportUnreachableProbability


   ID="$Measurement"
   POOLHANDLE="`echo Pool-$ID | md5sum | awk '{ print $1 }'`"

   echo "ID=$ID"
   echo "PoolHandle=$POOLHANDLE"


   # Get list of n nodes
   # $1 = Node list
   # $2 = n
   get_nodes ()
   {
      n=0
      NodeList=""
      CandidateList=$1
      if [ "$CandidateList" = "" ] ; then
         echo "ERROR: Empty node list $1!"
         exit 1
      fi

      while [ $n -lt $2 ] ; do
         for node in $CandidateList ; do
            NodeList="$NodeList $node "
            let n=$n+1
            if [ $n -ge $2 ] ; then
               return
            fi
         done
      done
   }


   get_nodes "$PRNodeList" $PRs             ; PRList=$NodeList
   get_nodes "$PENodeList" $PEs             ; PEList=$NodeList
   get_nodes "$PUNodeList" $PUs             ; PUList=$NodeList
   get_nodes "$AttackerNodeList" $Attackers ; AttackerList=$NodeList

   let ENRPPortBase=$PortBase+1024
   let ASAPPortBase=$PortBase+2048
   let AttackerPortBase=$PortBase+3072

   echo "Port Bases:"
   echo "   ENRP=$ENRPPortBase"
   echo "   ASAP=$ASAPPortBase"
   echo "   Attacker=$AttackerPortBase"

   PR_PROpt=""
   PE_PROpt=""
   PU_PROpt=""
   asapPort=$ASAPPortBase
   enrpPort=$ENRPPortBase
   MasterPR="`(echo $PRList | xargs -n1 echo | head -n1) 2>/dev/null`"
   for pr in $PRList ; do
      PR_PROpt="$PR_PROpt -peer=$pr:$enrpPort "
      PE_PROpt="$PE_PROpt -registrar=$pr:$asapPort "
      PU_PROpt="$PU_PROpt -registrar=$pr:$asapPort "
      let asapPort=$asapPort+1
      let enrpPort=$enrpPort+1
   done

   if [ "$UsePEMasterPR" = "true" ] ; then
      PE_PROpt=" -registrar=$MasterPR:$ASAPPortBase "
   fi
   if [ "$UsePUMasterPR" = "true" ] ; then
      PE_PROpt=" -registrar=$MasterPR:$ASAPPortBase "
   fi

   echo PR_PROpt=$PR_PROpt
   echo PE_PROpt=$PE_PROpt
   echo PU_PROpt=$PU_PROpt

   rm -f *.log *.cmd *.sca *.log.bz2 *.cmd.bz2 *.sca.bz2


   echo "`date` ###### Starting PRs ###################################################"
   id=1
   for node in `echo $PRList | xargs -n1 echo | sort -u` ; do
      rm -f $node-pr-start.cmd
   done
   asapPort=$ASAPPortBase
   enrpPort=$ENRPPortBase
   useTakeoverSuggestion=""
   if [ "$UseTakeoverSuggestion" = "true" ] ; then
      useTakeoverSuggestion="-supporttakeoversuggestion"
   fi
   for node in $PRList ; do
      uptime=$PRAllUptime
      downtime=$PRAllDowntime
      if [ $id -eq 1 ] ; then
         if [ "$PR1UptimeOverride" != "auto" ] ; then
            uptime=$PR1UptimeOverride
         fi
      fi

      echo "PR=$node   ASAP=$asapPort ENRP=$enrpPort   uptime=$uptime  downtime=$downtime   MiscOptions=$useTakeoverSuggestion"
# -peerheartbeatcycle=30000 -peermaxtimelastheard=61000
      $SERVICE_CONTROL_CMD restart   "$SSH_TOOL_CMD WriteCmdParallel $node-pr-start.cmd"  $node  $SSH_NODE $SSH_USER $SSH_KEY   $ID-$id $SERVICE_DIRECTORY registrar \
         "-loglevel=4 -logcolor=off -asap=[::]:$asapPort -enrp=[::]:$enrpPort -asapannounce=off -enrpannounce=off $useTakeoverSuggestion -maxbadpereports=$MaxBadPEReports -endpointkeepalivetransmissioninterval=$EKATransmissionInterval -endpointkeepalivetimeoutinterval=$EKATimeoutInterval -maxhrrate=$MaxHRRate -maxeurate=$MaxEURate $OPT_CSP $PR_PROpt -statsfile=$ID-pr-$id-stats.data.bz2 -statsinterval=1000 -actionlogfile=$ID-pr-$id-action.data.bz2 -uptime=$uptime -downtime=$downtime "-object=scenario.registrar[$id]" -scalar=$ID-pr-$id.sca"   "$SERVICE_PRE_PROGRAM"
      let id=$id+1
      let asapPort=$asapPort+1
      let enrpPort=$enrpPort+1
   done
   for node in `echo $PRList | xargs -n1 echo | sort -u` ; do
      $SSH_TOOL_CMD RunCmdParallel $node-pr-start.cmd  $node  $SSH_NODE $SSH_USER $SSH_KEY &
   done
   wait

   sleep 30

   echo "`date` ###### Starting attackers #############################################"
   id=1
   for node in `echo $AttackerList | xargs -n1 echo | sort -u` ; do
      rm -f $node-attacker-start.cmd
   done
   for node in $AttackerList ; do
      echo "A=$node"
      aid=0
      if [ $AttackIdentifier -ne 0 ] ; then
         let aid=$AttackIdentifier+$id-1
      fi
      let aport=$AttackerPortBase+$id
      $SERVICE_CONTROL_CMD restart   "$SSH_TOOL_CMD WriteCmdParallel $node-attacker-start.cmd"  $node  $SSH_NODE $SSH_USER $SSH_KEY   $ID-$id $SERVICE_DIRECTORY attacker \
         "-type=$AttackType -interval=$AttackInterval -port=$aport -identifier=$aid -policy=$Policy -loaddegoverride=$LoadDegOverride -poolhandle=$POOLHANDLE -reportunreachableprobability=$AttackReportUnreachableProbability -asapannounce=off $OPT_CSP $PE_PROpt"   "$SERVICE_PRE_PROGRAM"
      let id=$id+1
   done
   for node in `echo $AttackerList | xargs -n1 echo | sort -u` ; do
      $SSH_TOOL_CMD RunCmdParallel $node-attacker-start.cmd  $node  $SSH_NODE $SSH_USER $SSH_KEY &
   done
   wait

   sleep 6

   echo "`date`# ##### Starting PEs ###################################################"
   id=1
   for node in `echo $PEList | xargs -n1 echo | sort -u` ; do
      rm -f $node-pe-start.cmd
   done
   for node in $PEList ; do
      echo "PE=$node"
      $SERVICE_CONTROL_CMD restart   "$SSH_TOOL_CMD WriteCmdParallel $node-pe-start.cmd"  $node  $SSH_NODE $SSH_USER $SSH_KEY   $ID-$id $SERVICE_DIRECTORY server \
         "-loglevel=4 -logcolor=off -calcapp -capcapacity=1000000 -capmaxjobs=$MaxJobs -policy=$Policy -poolhandle=$POOLHANDLE -rereginterval=$ReregInterval -capcapacity=$Capacity -capkeepalivetimeoutinterval=$SKATimeoutInterval -capkeepalivetransmissioninterval=$SKATransmissionInterval -capcookiemaxcalculations=$CookieMaxCalculations -capcookiemaxtime=$CookieMaxTime -uptime=$PEAllUptime -downtime=$PEAllDowntime -capcleanshutdownprobability=$CleanShutdownProbability "-capobject=scenario.calcAppPoolElement[$id]" -capscalar=$ID-pe-$id.sca $PE_PROpt -asapannounce=off $OPT_CSP"   "$SERVICE_PRE_PROGRAM"
      let id=$id+1
   done
   v=1
   for node in `echo $PEList | xargs -n1 echo | sort -u` ; do
      $SSH_TOOL_CMD RunCmdParallel $node-pe-start.cmd  $node  $SSH_NODE $SSH_USER $SSH_KEY &
      let v=$v+1
   done
   wait

   sleep 6

   echo "`date` ###### Starting PUs ###################################################"
   id=1
   for node in `echo $PUList | xargs -n1 echo | sort -u` ; do
      rm -f $node-pu-start.cmd
   done
   for node in $PUList ; do
   echo "PU=$node"
      $SERVICE_CONTROL_CMD restart   "$SSH_TOOL_CMD WriteCmdParallel $node-pu-start.cmd"  $node  $SSH_NODE $SSH_USER $SSH_KEY   $ID-$id $SERVICE_DIRECTORY calcappclient \
         "-loglevel=4 -logcolor=off -jobsize=$JobSize -jobinterval=$JobInterval -keepalivetimeoutinterval=$SKATimeoutInterval -keepalivetransmissioninterval=$SKATransmissionInterval -poolhandle=$POOLHANDLE "-object=scenario.calcAppPoolUser[$id]" -scalar=$ID-pu-$id.sca $PE_PROpt -asapannounce=off $OPT_CSP"   "$SERVICE_PRE_PROGRAM"
      let id=$id+1
   done
   for node in `echo $PUList | xargs -n1 echo | sort -u` ; do
      $SSH_TOOL_CMD RunCmdParallel $node-pu-start.cmd  $node  $SSH_NODE $SSH_USER $SSH_KEY &
   done
   wait


   echo "`date` ###### Running test ###################################################"
   echo "Duration is $Duration seconds"
   echo "Now is `date`"
   sleep $Duration


   echo "`date` ###### Stopping PUs ###################################################"
   id=1
   for node in `echo $PUList | xargs -n1 echo | sort -u` ; do
      rm -f $node-pu-stop.cmd
   done
   for node in $PUList ; do
      $SERVICE_CONTROL_CMD stop  "$SSH_TOOL_CMD WriteCmdParallel $node-pu-stop.cmd"  $node  $SSH_NODE $SSH_USER $SSH_KEY   $ID-$id $SERVICE_DIRECTORY calcappclient ""   "$SERVICE_PRE_PROGRAM"
      let id=$id+1
   done
   for node in `echo $PUList | xargs -n1 echo | sort -u` ; do
      $SSH_TOOL_CMD RunCmdParallel $node-pu-stop.cmd  $node  $SSH_NODE $SSH_USER $SSH_KEY &
   done
   wait


   echo "`date` ###### Stopping PEs ###################################################"
   id=1
   for node in `echo $PEList | xargs -n1 echo | sort -u` ; do
      rm -f $node-pe-stop.cmd
   done
   for node in $PEList ; do
      $SERVICE_CONTROL_CMD stop  "$SSH_TOOL_CMD WriteCmdParallel $node-pe-stop.cmd"  $node  $SSH_NODE $SSH_USER $SSH_KEY   $ID-$id $SERVICE_DIRECTORY server ""   "$SERVICE_PRE_PROGRAM"
      let id=$id+1
   done
   for node in `echo $PEList | xargs -n1 echo | sort -u` ; do
      $SSH_TOOL_CMD RunCmdParallel $node-pe-stop.cmd  $node  $SSH_NODE $SSH_USER $SSH_KEY &
   done
   wait


   echo "`date` ###### Stopping attackers #############################################"
   id=1
   for node in `echo $AttackerList | xargs -n1 echo | sort -u` ; do
      rm -f $node-attacker-stop.cmd
   done
   for node in $AttackerList ; do
      $SERVICE_CONTROL_CMD stop  "$SSH_TOOL_CMD WriteCmdParallel $node-attacker-stop.cmd"  $node  $SSH_NODE $SSH_USER $SSH_KEY   $ID-$id $SERVICE_DIRECTORY attacker ""   "$SERVICE_PRE_PROGRAM"
      let id=$id+1
   done
   for node in `echo $AttackerList | xargs -n1 echo | sort -u` ; do
      $SSH_TOOL_CMD RunCmdParallel $node-attacker-stop.cmd  $node  $SSH_NODE $SSH_USER $SSH_KEY &
   done
   wait


   echo "`date` ###### Stopping PRs ###################################################"
   id=1
   for node in `echo $PRList | xargs -n1 echo | sort -u` ; do
      rm -f $node-pr-stop.cmd
   done
   for node in $PRList ; do
      $SERVICE_CONTROL_CMD stop  "$SSH_TOOL_CMD WriteCmdParallel $node-pr-stop.cmd"  $node  $SSH_NODE $SSH_USER $SSH_KEY   $ID-$id $SERVICE_DIRECTORY registrar ""   "$SERVICE_PRE_PROGRAM"
      let id=$id+1
   done
   for node in `echo $PRList | xargs -n1 echo | sort -u` ; do
      $SSH_TOOL_CMD RunCmdParallel $node-pr-stop.cmd  $node  $SSH_NODE $SSH_USER $SSH_KEY &
   done
   wait


   echo "`date` ###### Downloading results ############################################"
   allNodes="`
   (
      for pr in $PRList ; do
         echo $pr
      done
      for pe in $PEList ; do
         echo $pe
      done
      for pu in $PUList ; do
         echo $pu
      done
   ) | xargs -n1 echo | sort -u`"

   for node in $allNodes ; do
      echo "N=$node"
      scp -qC -i $SSH_KEY -oConnectTimeout=5 -oConnectionAttempts=4 -oStrictHostKeyChecking=no -oPasswordAuthentication=no "$SSH_USER@$node:$SERVICE_DIRECTORY/$ID-*.*" . &
   done
   wait
   bzip2 *.cmd *.sca
fi

touch status.txt
