#!/bin/sh
# Copyright (c) 2002-17 Peter Guentert. All rights reserved.

# Startup script
# Peter Guentert, 21-01-2002
# Modified version including job submission, Peter Guentert, 03-05-2011
# Simplified version without indirect gateway, Peter Guentert, 28-02-2014


# ------ Serial number -----

serial=

# ------ Program name -----

PROG=CYANA
prog=cyana
exe=${prog}exe

# ------ Command line options ------

params=$*
cmd="\$exefile"
options=a:c:in:q:r:R:St:v:Vxh

optind=1
usage=""
eval par=\$$optind
while true; do
  case $par in
  --) optind=`expr $optind + 1`; break;;
  -?*) option=`echo X$par | sed 's/^X-\(.\).*/\1/'`
       if [ `echo $options | sed 's/.*'$option'.*/+/'` != "+" ]; then
         echo "$0: -$option: unknown option"; usage=1
       elif [ `echo $options | sed 's/.*'$option':.*/+/'` = "+" ]; then
         optind=`expr $optind + 1`
         eval $option=\$$optind
       else
         eval $option=1
       fi
       par=`echo X$par | sed 's/^X-./-/'`
       if [ "$par" = "-" ]; then
         optind=`expr $optind + 1`
         eval par=\$$optind
       fi
       ;;
  *) break;;
  esac
done
shift `expr $optind - 1`

if [ "$h" ]; then usage=1; fi
if [ "$usage" ]; then
  echo "Usage: $prog -hciStvx parameters ...         (interactive)"
  echo "       $prog -hanqrStvx macro parameters ... (batch job)"
  echo
  echo "       -h            help"
  echo "       -a file       start batch job only after <file> exists"
  echo "       -c command    command to start program (must contain \$exefile)"
  echo "       -i            start interactive cyanashell with history etc."
  echo "       -n nproc      number of processors for batch job"
  echo "       -q queue      batch queue, or - to run in background"
  echo "       -r host       use remote host"
  echo "       -S            run in safe mode, i.e. within one directory"
  echo "       -t system     explicit system type"
  echo "       -v version    explicit program version"
  echo "       -V            display binary version information"
  echo "       -x            display name of executable without executing it"
  echo "       -s            run CYANA interactive shell"
  exit 2
fi
if [ "$c" ]; then cmd="$c"; fi
sys=$t
mode=$x
safe=$S
remote=$r
return=$R
version=$V

if [ "$return" = "-" ]; then remote=""; return=""; fi
#host=`hostname -s`
host=`uname -n | awk -F. '{print $1}'`
dir=`pwd`
basedir=`basename $dir`


# ------ Full path name of program directory ------

f=$0
if [ `echo $f | sed 's,^/.*,+,'` != "+" ]; then f=`which $0`; fi
odir=`pwd`
while [ "`ls -l $f | awk '{ print substr($1,1,1) }'`" = "l" ]; do
  l=`ls -l $f | awk '{ i = NF; print $i }'`
  cd `dirname $f`; cd `dirname $l`; l=`pwd`/`basename $l`
  f=$l
done
cd `dirname $f`
libdir=`pwd | sed 's,/src$,,'`
cd "$odir"


# ------ Interactive shell ------

if [ "$i" ]; then
  error=0

  perl -MTerm::Shell -e '' 2> /dev/null
  if [ $? != 0 ]; then
    echo "ERROR: Please install Perl library 'Term::Shell' (e.g. using 'cpan install Term::Shell')"
    error=1
  fi

  perl -MTerm::ReadLine -e '' 2> /dev/null
  if [ $? != 0 ]; then
    echo "ERROR: Please install Perl library 'Term::ReadLine::Gnu' (e.g. using 'cpan install Term::ReadLine::Gnu')"
    error=1
  fi

  perl -MTerm::ReadKey -e '' 2> /dev/null
  if [ $? != 0 ]; then
    echo "ERROR: Please install Perl library 'Term::ReadKey' (e.g. using 'cpan install Term::ReadKey')"
    error=1
  fi

  if [ $error != 0 ]; then exit 1; fi

  #perl -MTerm::Shell -e '' ; echo $?
  #echo $libdir/cyanashell
  exec $libdir/cyanashell -path $libdir
fi


# ------ Call a different version of the program ------

if [ "$v" ]; then
  if [ `echo $v | sed 's,^/.*,+,'` != "+" ]; then v=$libdir/$v; fi
  if [ `echo $v | sed 's,.*/'$prog',+,'` != "+" ]; then v=$v/$prog; fi
  params=`echo $params | sed 's/-v  *[^ ][^ ]*//'`
  exec $v $params
fi


# ------ Macro name ------

start=1
if [ "$a" -o "$n" -o "$q" ]; then
  start=
  if [ $# -lt 1 ]; then
    echo "Missing macro file."
    echo "params=$params"
    exit 2
  fi
  macro=`echo $1 | sed 's/\.cya$//'`
  if [ ! -f $macro.cya ]; then
    echo "Macro file $macro.cya not found."
    exit 2
  fi
  shift
  name=`basename $macro`
fi


# ====== Return results from remote host ======

if [ "$return" ]; then
  remote=`echo $remote | awk -F: '{ print $1 }'`
  echo "#!/bin/sh
cd $dir
t=0
while [ ! -f $name.out ]; do
  sleep 1
  t=\`expr \$t + 1\`
  if [ \$t -gt 60 ]; then echo Timeout > $name.out; fi
done
sleep 2
rm -f $name.job
chmod g+r $name.out" > ../returndata
  if [ `echo $return | sed 's/^.*\.tgz$/.tgz/'` != ".tgz" ]; then
    echo "rsync -a $dir/ $return" >> ../returndata
  else
    echo "cd ..
tar zcf $basedir.tgz $basedir
rsync -a $basedir.tgz $return
#rm -rf $basedir.tgz
" >> ../returndata
  fi
  echo "if [ \$? -eq 0 ]; then rm -rf `dirname $dir`; fi" >> ../returndata
  chmod +x ../returndata


# ====== Start batch job on remote host ======

elif [ "$remote" ]; then
  rm -f $macro.out
  prog=`echo $remote | awk -F: '{ if (NF==1) print "cyana"; else print $2 }'`
  if [ "$prog" = "" ]; then prog=cyana; fi
  remote=`echo $remote | awk -F: '{ print $1 }'`
  #echo "prog=$prog remote=$remote"
  tmp=c$$
  #echo "mkdir -p cyanatmp/$tmp;
  ssh $remote "mkdir -p cyanatmp/$tmp
               chdir cyanatmp/$tmp
               rsync -a $host:$dir .
               chdir $basedir
               #echo \"$prog -R $host:$dir $params\"
               $prog -R $host:$dir $params"
  if [ $? -eq 0 ]; then echo "Job submitted remotely on host $remote."; fi
  exit 0
fi

#start=1; if [ "$a" -o "$n" -o "$q" ]; then start=; fi


# ====== Start the program ======

if [ "$start" ]; then

  # ------ Environment variables passed to executable ------

  eval ${PROG}ARG='$*'
  export ${PROG}ARG
  eval ${PROG}LIB=$libdir
  export ${PROG}LIB

  if [ "$safe" ]; then
    CYANAINIT=initsafe; export CYANAINIT
  fi
  if [ "$CYANASERIAL" ]; then serial=$CYANASERIAL; fi
  CYANASERIAL=$serial; export CYANASERIAL


  # ------ Start executable ------

  if [ -d $libdir/src ]; then
    exe=$libdir/src/$exe
  else
    exe=$libdir/$exe
  fi
  sysid=`$libdir/etc/identify`
  os=`echo $sysid | awk -F- '{ print $1 }'`
  osbase=`echo $os | sed -e 's/32$//' -e 's/64$//'`
  if [ "$sys" = "" ]; then
    sys=$sysid
    sysbase=`echo $sys | sed -e 's/32-/-/' -e 's/64-/-/'`
#    files="$exe.$sys* $exe.$sysbase* $exe.$os-gfortran $exe.$osbase-gfortran $exe.$os* $exe.$osbase* $exe.*"
    files="$exe.$os-intel $exe.$osbase-intel $exe.$os-gfortran $exe.$osbase-gfortran $exe.$os* $exe.$osbase* $exe.*"
  else
    files=
    while [ "$sys" ]; do
      sys1=`echo $sys | sed 's/,.*$//'`
      sysbase=`echo $sys1 | sed -e 's/32-/-/' -e 's/64-/-/'`
      files="$files $exe.$sys1 $exe.$os-$sys1 $exe.$osbase-$sys1 $exe.*-$sys1"
      sys=`echo $sys | sed -e 's/^[^,]*$//' -e 's/^[^,]*,//'`
    done
  fi
#  echo "files=$files"
  for exefile in $files; do
    if [ -x $exefile ]; then
      if [ "$mode" ]; then
        echo $exefile; exit 0
      elif [ "$version" ]; then
        echo "Executable: $exefile"
        echo "Attributes: `ls -l $exefile | sed 's/ [^ ]*$//'`"
        echo "Filetype  : `file $exefile | sed 's/[^ ]*  *//'`"
        echo "MD5 sum   : `md5sum $exefile | awk '{print $1}'`"
        echo "License   : `md5sum $libdir/etc/license | awk '{print $1}'`"
        echo "Host      : `ping -c 1 $host | awk '/^PING / { print $2,$3}'`"
        echo "Hosttype  : `uname -a`"
        exit 0
      else
        if [ "`echo $cmd | grep -c '\$exefile'`" -gt 0 ]; then
          eval exec $cmd
        else
          eval exec $cmd $exefile
        fi
      fi
    fi
  done
  echo "No executable found in \"`dirname $exe`\"."
  exit 1
fi


# ====== Start CYANA batch job ======

dir=`pwd`
nproc=1
ncpu=1
queue=-
opt=
submit='nice $name.job >$name.out &'
nodes=
after=


# ------ Function to sort hostfile for OpenMPI by increasing load ------

sortnodes() {

nodes=`sed 's/#.*$//' $1 | awk 'NF>=1 { printf(" %s",$1) }'`
(for node in $nodes; do echo "$node: `ssh $node uptime`"; done; cat $1) |\
sed -e 's/: .*load average: / load = /' -e 's/, [^,]*, [^,]*$//' -e 's/#.*$//' |\
awk '$2=="load" { load[$1]=$4; next }
     NF>=1 { print load[$1], $0 }' |\
sort -n |\
sed 's/\(^[0-9][.0-9]* \)\(.*$\)/\2 \# load \1/'
}


# ------ Defaults for specific computers ------

# Add to this section a new entry that fits your compute server and batch system.

case $host in
  euler*)
    cluster=euler
    queue=normal.4h
    nproc=10
    ;;
  honshu | hokkaido | kyushu | node*)
    cluster=bmrz
    queue=batch
    nproc=20
    ;;
  endeavour | cnode*)
    cluster=tmu
    queue=default
    nproc=20
    ncpu=12
    ;;
  also | dhgh | guri1 | guri2 | guri3 | juge | lufr | prku | rabo | saro | kreide | stift)
    cluster=riek
    queue=bnmr
    nproc=10
    ;;
#  kume | odaiba)
#    cluster=background
#    nproc=10
#    ;;
  *)
    cluster=-
    nproc=4
    ;;
esac

if [ "$a" ]; then after="$a"; fi
if [ "$n" ]; then nproc="$n"; fi
if [ "$q" ]; then queue="$q"; fi

case $cluster in
  -) # If no cluster defined, assume a shared-memory multiprocessor machine without batch system
    submit="./$name.job > $name.out &"
    ;;
  demo) # Demo (explanation of options)
    opt="-t demo-mpi -c 'mpirun -np NPROCPLUSONE'"   # options to start CYANA
    submit='qsub $name.job'                          # command to submit batch job
    ;;
  background) # Do not use batch system; run in background; start on least loaded nodes
    nohostfile=y
    for hostfile in hostfile $HOME/.openmpi/hostfile; do
      if [ -f $hostfile ]; then
        sortnodes $hostfile > openmpi.hosts
        opt="-t intel-openmpi,gfortran-openmpi -c 'mpirun --hostfile openmpi.hosts -x CYANALIB -np NPROCPLUSONE --mca btl tcp,self'"
        nohostfile=""
        break
      fi
    done
    if [ "$nohostfile" ]; then
      opt="-t intel-openmpi,gfortran-openmpi -c 'mpirun -x CYANALIB -np NPROCPLUSONE --mca btl tcp,self'"
    fi
    submit="./$name.job > $name.out &"
    ;;
  euler)
    opt="-t intel-openmpi,gfortran-openmpi -c 'mpirun -np NPROCPLUSONE'"
    submit="bsub -q $queue -n NNODES -o $dir/$name.out $dir/$name.job"
    ;;
  riek)
#   opt="-t intel-openmpi,gfortran-openmpi -c 'mpirun -bootstrap slurm -n NPROCPLUSONE'"
#   submit="sbatch -p $queue -J $name -N 9 -n $n --ntasks-per-node=4 -o $dir/$name.out $dir/$name.job"
    opt="-t intel-openmpi,gfortran-openmpi -c 'srun --mpi=pmi2 -n NPROCPLUSONE'"
    submit="sbatch -p $queue -J $name -n NPROCPLUSONE -o $dir/$name.out $dir/$name.job"
    ;;
  *)
    opt="-t intel-openmpi,gfortran-openmpi -c 'mpirun --hostfile \$PBS_NODEFILE -x CYANALIB -np NPROCPLUSONE --mca btl tcp,self'"
    submit="qsub -q $queue -N $name -j oe -o $dir/$name.out -l nodes=NNODES $name.job"
    ;;
esac


# ------ Start options and submit command ------

nproc1=`expr $nproc + 1`
nnodes=`echo $nproc1 $ncpu |\
        awk '{ n=int($1/$2)
               if (n==0) print "1:ppn=" $1
               else if (n*$2==$1) print n ":ppn=" $2
               else print n ":ppn=" $2 "+1:ppn=" $1-n*$2 }'`

opt=`echo $opt | sed "s/NPROCPLUSONE/$nproc1/"`
submit=`echo $submit | sed -e "s/NPROCPLUSONE/$nproc1/" -e "s/NNODES/$nnodes/"`
echo "opt=$opt"
echo "submit=$submit"


# ------ Write job script ------


echo "#!/bin/sh
cd `pwd`
date +'%d-%b-%Y %H:%M:%S'
t0=\`date +%s\`
uname -a
#head -1 /etc/issue
grep '<< EOF$' $name.job | sed 's/ *<< EOF//'
$libdir/$prog -x $opt
echo nproc=$nproc
#printenv | fgrep PBS
#if [ \"\$PBS_NODEFILE\" ]; then echo PBS_NODEFILE:; cat \$PBS_NODEFILE; fi
$libdir/$prog $safe $opt << EOF
erract:=abort
nproc=$nproc
$macro $*
quit
EOF
echo
t1=\`date +%s\`; t1=\`expr \$t1 - \$t0\`
echo \"Total elapsed time: \$t1 s\"
date +'%d-%b-%Y %H:%M:%S'
" > $name.job
if [ "$return" ]; then
  dir=`pwd`
  echo "ssh -f $host \"`dirname $dir`/returndata &\"" >> $name.job
fi
chmod +x $name.job


# ------ Submit job ------

submitjob() {
#echo "$submit"
eval $submit
case "$queue" in
-) echo "Background job $name started on $nproc processors.";;
*) echo "Job $name submitted to queue $queue, $nproc processors.";;
esac
}

rm -f $name.out
if [ "$after" ]; then
  echo "Job $name will start when file $after exists."
  (while [ ! -f $after ]; do sleep 10; done; submitjob) &
else
  submitjob
fi
