#!/bin/sh
# Copyright (c) 2002-20 Peter Guentert. All rights reserved.

# Startup script
# Peter Guentert, 21-01-2002
# Modified version including job submission, Peter Guentert, 03-05-2011
# Simplified version without indirect gateway, Peter Guentert, 28-02-2014


# ------ Serial number -----

serial=

# ------ Program name -----

PROG=CYANA
prog=cyana
exe=${prog}exe

# ------ Command line options ------

params=$*
cmd="\$exefile"
options=a:c:C:giln:opq:r:R:St:Tv:Vxh

optind=1
usage=""
eval par=\$$optind
while true; do
  case $par in
  --) optind=`expr $optind + 1`; break;;
  -?*) option=`echo X$par | sed 's/^X-\(.\).*/\1/'`
       if [ `echo $options | sed 's/.*'$option'.*/+/'` != "+" ]; then
         echo "$0: -$option: unknown option"; usage=1
       elif [ `echo $options | sed 's/.*'$option':.*/+/'` = "+" ]; then
         optind=`expr $optind + 1`
         eval $option=\$$optind
       else
         eval $option=1
       fi
       par=`echo X$par | sed 's/^X-./-/'`
       if [ "$par" = "-" ]; then
         optind=`expr $optind + 1`
         eval par=\$$optind
       fi
       ;;
  *) break;;
  esac
done
shift `expr $optind - 1`

if [ "$h" ]; then usage=1; fi
if [ "$usage" ]; then
  echo "Usage: $prog -hciStvx parameters ...             (interactive)"
  echo "       $prog -hCaglnopqrStTvx macro parameters ... (batch job)"
  echo
  echo "       -h            help"
  echo "       -a file       start batch job only after <file> exists"
  echo "       -c command    command to start program (must contain \$exefile)"
  echo "       -C cluster    execute on given cluster system"
  echo "       -g            in passive mode, return data from unfinished job"
  echo "       -i            start interactive cyanashell with history etc."
  echo "       -l            in passive mode, directory listing of unfinished job"
  echo "       -n nproc      number of processors for batch job"
  echo "       -o            in passive mode; overwrite local directory"
  echo "       -p            passive mode; no remote access by the remote host"
  echo "       -q queue      batch queue, or - to run in background"
  echo "       -r host       use remote host"
  echo "       -S            run in safe mode, i.e. within one directory"
  echo "       -t system     explicit system type"
  echo "       -T            use ssh reverse tunneling to return data from remote host"
  echo "       -v version    explicit program version"
  echo "       -V            display binary version information"
  echo "       -x            display name of executable without executing it"
  exit 2
fi
if [ "$c" ]; then cmd="$c"; fi
cluster=$C
sys=$t
mode=$x
safe=$S
remote=$r
passive=$p
list=$l
getdata=$g
if [ "$o" ]; then overwrite="--delete-after"; fi
return=$R
tunnel=$T
version=$V

if [ "$return" = "-" ]; then remote=""; return=""; fi
#host=`hostname -s`
#host=`uname -n | awk -F. '{print $1}'`
host=$USER@`uname -n | awk -F. '{print $1}'`
dir=`pwd`
basedir=`basename $dir`
tunnelport=19999
tunneltime=7200


# ------ CYANASERVER environment variable ------

if [ "$remote" = "" -a "$CYANASERVER" ]; then
  if [ "$n" -o "$p" ]; then
    remote=`echo "$CYANASERVER" | awk '{ print $1 }'`
    if [ "$passive" = "" ]; then
      passive=`echo "$CYANASERVER" | awk ' { if ($2=="-p") print 1 }'`
    fi
  fi
fi
#echo "remote=$remote"
#echo "passive=$passive"


# ------ Full path name of program directory ------

f=$0
if [ `echo $f | sed 's,^/.*,+,'` != "+" ]; then f=`which $0`; fi
odir=`pwd`
while [ "`ls -l $f | awk '{ print substr($1,1,1) }'`" = "l" ]; do
  l=`ls -l $f | awk '{ i = NF; print $i }'`
  cd `dirname $f`; cd `dirname $l`; l=`pwd`/`basename $l`
  f=$l
done
cd `dirname $f`
libdir=`pwd | sed 's,/src$,,'`
cd "$odir"


# ------ Interactive shell ------

if [ "$i" ]; then
  error=0

  perl -MTerm::Shell -e '' 2> /dev/null
  if [ $? != 0 ]; then
    echo "ERROR: Please install Perl library 'Term::Shell' (e.g. using 'cpan install Term::Shell')"
    error=1
  fi

  perl -MTerm::ReadLine -e '' 2> /dev/null
  if [ $? != 0 ]; then
    echo "ERROR: Please install Perl library 'Term::ReadLine::Gnu' (e.g. using 'cpan install Term::ReadLine::Gnu')"
    error=1
  fi

  perl -MTerm::ReadKey -e '' 2> /dev/null
  if [ $? != 0 ]; then
    echo "ERROR: Please install Perl library 'Term::ReadKey' (e.g. using 'cpan install Term::ReadKey')"
    error=1
  fi

  if [ $error != 0 ]; then exit 1; fi

  #perl -MTerm::Shell -e '' ; echo $?
  #echo $libdir/cyanashell
  exec perl $libdir/etc/cyanashell.pl -path $libdir
fi


# ------ Call a different version of the program ------

if [ "$v" ]; then
  if [ `echo $v | sed 's,^/.*,+,'` != "+" ]; then v=$libdir/$v; fi
  if [ `echo $v | sed 's,.*/'$prog',+,'` != "+" ]; then v=$v/$prog; fi
  params=`echo $params | sed 's/-v  *[^ ][^ ]*//'`
  exec $v $params
fi


# ====== Fetch results from remote host (passive mode) ======

if [ "$passive" -a $# -lt 1 ]; then
  if [ "$remote" = "" ]; then
    echo "No remote host given (missing -r option)."
    exit 2
  fi
  remote=`echo $remote | awk -F: '{ print $1 }'`
  tmp=t$$
  opt=""
  if [ "$list" ]; then opt="-l"; fi
  if [ "$getdata" ]; then opt="$opt -g"; fi
  ssh $remote "cd cyanatmp; ./returnpassive $opt $host:$dir $tmp" | tee $tmp.err
  grep '^No data on remote server.' $tmp.err > /dev/null
  if [ $? -eq 0 ]; then rm -f $tmp.err; exit 2; fi
  grep '^Unfinished job.' $tmp.err > /dev/null
  res=$?
  rm -f $tmp.err
  if [ $res -eq 0 ]; then
    if [ "$getdata" ]; then
      rsync -a $overwrite $remote:cyanatmp/$tmp/ .
      if [ $? -eq 0 ]; then echo "Intermediate results fetched from host $remote."; fi
    fi
    exit 2
  fi
  rsync -a $overwrite $remote:cyanatmp/$tmp/ .
  res=$?
  ssh $remote "cd cyanatmp; ./returnpassive -c $host:$dir $tmp"
  if [ $res -eq 0 ]; then echo "Results fetched from host $remote."; fi
  exit 0
fi


# ------ Macro name ------

start=1
if [ "$a" -o "$n" -o "$q" ]; then
  start=
  if [ $# -lt 1 ]; then
    echo "Missing macro file."
    echo "params=$params"
    exit 2
  fi
  macro=`echo $1 | sed 's/\.cya$//'`
  if [ ! -f $macro.cya ]; then
    echo "Macro file $macro.cya not found."
    exit 2
  fi
  shift
  name=`basename $macro`
fi


# ====== Return results from remote host ======

if [ "$return" ]; then

  # ------ Passive mode ------

  if [ "$passive" ]; then
    echo '#!/bin/sh
#pwd
#echo "arg1=$1"
#echo "arg2=$2"
cd $HOME/cyanatmp
# clean cyanatmp directory
if [ "$1" = "-c" ]; then
  shift
  d=`readlink $2`
  d=`dirname $d`
  rm -rf $d
  # remove symbolic links to non-existant directories
  for f in `\ls t[0-9]* 2>/dev/null`; do
    d=`readlink $f`
#    echo "f=$f, d=$d"
    if [ ! -d $d ]; then rm -f $f; fi
  done
  exit 0
fi
# Find data to return, purge outdated jobs
list=""; if [ "$1" = "-l" ]; then list=1; shift; fi
getdata=""; if [ "$1" = "-g" ]; then getdata=1; shift; fi
n=0
for f in `\ls -t */FROM */DONE 2>/dev/null`; do
  r=`cat $f`
  if [ "$r" != "$1" ]; then continue; fi
  n=`expr $n + 1`
  d=`dirname $f`
  if [ $n -ne 1 ]; then
    rm -rf $d
  elif [ `basename $f` = "DONE" ]; then
    ln -s $d/`basename $1` $2
  else
    if [ "$list" ]; then (cd $d/`basename $1`; ls -l); fi
    if [ "$getdata" ]; then ln -s $d/`basename $1` $2; fi
    echo "Unfinished job."
  fi
done
if [ $n -eq 0 ]; then
  echo "No data on remote server."
elif [ $n -gt 1 ]; then
  n=`expr $n - 1`
  echo "Data from $n outdated jobs purged."
fi
' > ../../returnpassive
  chmod +x ../../returnpassive

  # ------ Active mode ------

  else
    remote=`echo $remote | awk -F: '{ print $1 }'`
    echo "#!/bin/sh
cd $dir
t=0
while [ ! -f $name.out ]; do
  sleep 1
  t=\`expr \$t + 1\`
  if [ \$t -gt 60 ]; then echo Timeout > $name.out; fi
done
sleep 2
rm -f $name.job
chmod g+r $name.out" > ../returndata
    port=
    if [ `echo $return | awk -F: '{print $1}'` = "localhost" ]; then port='-e "ssh -p '$tunnelport'"'; fi
    if [ `echo $return | sed 's/^.*\.tgz$/.tgz/'` != ".tgz" ]; then
      echo "rsync $port -a $dir/ $return" >> ../returndata
    else
      echo "cd ..
tar zcf $basedir.tgz $basedir
rsync $port -a $basedir.tgz $return
#rm -rf $basedir.tgz
" >> ../returndata
    fi
    echo "if [ \$? -eq 0 ]; then rm -rf `dirname $dir`; fi" >> ../returndata
    chmod +x ../returndata
  fi

# ====== Start batch job on remote host ======

elif [ "$remote" ]; then
  rm -f $macro.out
  prog=`echo $remote | awk -F: '{ if (NF==1) print "cyana"; else print $2 }'`
  if [ "$prog" = "" ]; then prog=cyana; fi
  remote=`echo $remote | awk -F: '{ print $1 }'`
  #echo "prog=$prog remote=$remote"
  tmp=c$$
  #echo "mkdir -p cyanatmp/$tmp;
  returnhost=$host
  if [ "$passive" ]; then
    ssh $remote "mkdir -p cyanatmp/$tmp"
    rsync -a $dir $remote:cyanatmp/$tmp
    ssh $remote "cd cyanatmp/$tmp/$basedir
                 #echo \"$prog -R $returnhost:$dir $params\"
                 $prog -p -R $returnhost:$dir $params"
  else
    if [ "$tunnel" ]; then
      ssh -f -R $tunnelport:localhost:22 $remote sleep $tunneltime
      if [ $? -eq 0 ]; then
        echo "Tunnel to host $remote opened for $tunneltime s using port $tunnelport."
      else
        echo "Cannot open reverse ssh tunnel to host $remote."
        exit 2
      fi
      returnhost=localhost
    fi
    ssh $remote "mkdir -p cyanatmp/$tmp
                 cd cyanatmp/$tmp
                 rsync -a $returnhost:$dir .
                 cd $basedir
                 #echo \"$prog -R $returnhost:$dir $params\"
                 $prog -R $returnhost:$dir $params"
  fi
  if [ $? -eq 0 ]; then
    echo "Job submitted remotely on host $remote."
    if [ "$passive" ]; then
      echo "Passive mode: Use the following commands to retrieve results:"
      echo "cd `pwd`; cyana -p -r $remote:$prog"
    fi
    exit 0
  else
    echo "Job submission to remote host $remote failed."
    exit 2
  fi
fi

#start=1; if [ "$a" -o "$n" -o "$q" ]; then start=; fi


# ====== Start the program ======

if [ "$start" ]; then

  # ------ Environment variables passed to executable ------

  eval ${PROG}ARG='$*'
  export ${PROG}ARG
  eval ${PROG}LIB=$libdir
  export ${PROG}LIB

  if [ "$safe" ]; then
    CYANAINIT=initsafe; export CYANAINIT
  fi
  if [ "$CYANASERIAL" ]; then serial=$CYANASERIAL; fi
  CYANASERIAL=$serial; export CYANASERIAL


  # ------ Start executable ------

  if [ -d $libdir/src ]; then
    exe=$libdir/src/$exe
  else
    exe=$libdir/$exe
  fi
  sysid=`$libdir/etc/identify`
  os=`echo $sysid | awk -F- '{ print $1 }'`
  osbase=`echo $os | sed -e 's/32$//' -e 's/64$//'`
  if [ "$sys" = "" ]; then
    sys=$sysid
    sysbase=`echo $sys | sed -e 's/32-/-/' -e 's/64-/-/'`
#    files="$exe.$sys* $exe.$sysbase* $exe.$os-gfortran $exe.$osbase-gfortran $exe.$os* $exe.$osbase* $exe.*"
    files="$exe.$os-intel $exe.$osbase-intel $exe.$os-gfortran $exe.$osbase-gfortran $exe.$os* $exe.$osbase* $exe.*"
  else
    files=
    while [ "$sys" ]; do
      sys1=`echo $sys | sed 's/,.*$//'`
      sysbase=`echo $sys1 | sed -e 's/32-/-/' -e 's/64-/-/'`
      files="$files $exe.$sys1 $exe.$os-$sys1 $exe.$osbase-$sys1 $exe.*-$sys1"
      sys=`echo $sys | sed -e 's/^[^,]*$//' -e 's/^[^,]*,//'`
    done
  fi
#  echo "files=$files"
  for exefile in $files; do
    if [ -x $exefile ]; then
      if [ "$mode" ]; then
        echo $exefile; exit 0
      elif [ "$version" ]; then
        echo "Executable: $exefile"
        echo "Attributes: `ls -l $exefile | sed 's/ [^ ]*$//'`"
        echo "Filetype  : `file $exefile | sed 's/[^ ]*  *//'`"
        echo "MD5 sum   : `md5sum $exefile | awk '{print $1}'`"
        echo "License   : `md5sum $libdir/etc/license | awk '{print $1}'`"
        echo "Host      : `ping -c 1 $host | awk '/^PING / { print $2,$3}'`"
        echo "Hosttype  : `uname -a`"
        exit 0
      else
        ulimit -Ss unlimited
        if [ "`echo $cmd | grep -c '\$exefile'`" -gt 0 ]; then
          eval exec $cmd
        else
          eval exec $cmd $exefile
        fi
      fi
    fi
  done
  echo "No executable found in \"`dirname $exe`\"."
  exit 1
fi


# ====== Start CYANA batch job ======

dir=`pwd`
nproc=1
ncpu=1
queue=-
opt=
submit='nice $name.job >$name.out &'
nodes=
after=


# ------ Function to sort hostfile for OpenMPI by increasing load ------

sortnodes() {

nodes=`sed 's/#.*$//' $1 | awk 'NF>=1 { printf(" %s",$1) }'`
(for node in $nodes; do echo "$node: `ssh $node uptime`"; done; cat $1) |\
sed -e 's/: .*load average: / load = /' -e 's/, [^,]*, [^,]*$//' -e 's/#.*$//' |\
awk '$2=="load" { load[$1]=$4; next }
     NF>=1 { print load[$1], $0 }' |\
sort -n |\
sed 's/\(^[0-9][.0-9]* \)\(.*$\)/\2 \# load \1/'
}


# ------ Defaults for specific computers ------

# Add to this section a new entry that fits your compute server and batch system.

if [ "$cluster" = "" ]; then cluster=$CYANACLUSTER; fi
if [ "$cluster" = "" ]; then
  hostname=`echo $host | sed 's/.*@//'`
  case $hostname in
    euler* | eu-login*)
      cluster=euler
      ;;
    honshu | hokkaido | kyushu | node*)
      cluster=bmrz
      ;;
    endeavour | cnode*)
      cluster=tmu
      ;;
    guri*)
      cluster=riek
      ;;
  #  kume | odaiba)
  #    cluster=background
  #    ;;
    *)
      cluster=-
      ;;
  esac
fi

case $cluster in
  euler)
    queue=normal.4h
    nproc=10
    ;;
  bmrz)
    queue=batch
    nproc=20
    ;;
  tmu)
    queue=default
    nproc=20
    ncpu=12
    ;;
  riek)
    queue=cpu
    nproc=10
    ;;
  *)
    nproc=4
    ;;
esac

if [ "$a" ]; then after="$a"; fi
if [ "$n" ]; then nproc="$n"; fi
if [ "$q" ]; then queue="$q"; fi

case $cluster in
  -) # If no cluster defined, assume a shared-memory multiprocessor machine without batch system
    submit="./$name.job > $name.out &"
    ;;
  demo) # Demo (explanation of options)
    opt="-t demo-mpi -c 'mpirun -np NPROCPLUSONE'"   # options to start CYANA
    submit='qsub $name.job'                          # command to submit batch job
    ;;
  background) # Do not use batch system; run in background; start on least loaded nodes
    nohostfile=y
    for hostfile in hostfile $HOME/.openmpi/hostfile; do
      if [ -f $hostfile ]; then
        sortnodes $hostfile > openmpi.hosts
        opt="-t intel-openmpi,gfortran-openmpi -c 'mpirun --hostfile openmpi.hosts -x CYANALIB -np NPROCPLUSONE --mca btl tcp,self'"
        nohostfile=""
        break
      fi
    done
    if [ "$nohostfile" ]; then
      opt="-t intel-openmpi,gfortran-openmpi -c 'mpirun -x CYANALIB -np NPROCPLUSONE --mca btl tcp,self'"
    fi
    submit="./$name.job > $name.out &"
    ;;
  euler)
    opt="-t intel-openmpi,gfortran-openmpi -c 'mpirun -np NPROCPLUSONE'"
    #submit="bsub -q $queue -n NNODES -o $dir/$name.out $dir/$name.job"
    submit="sbatch -p $queue -J $name -n NPROCPLUSONE -o $dir/$name.out $dir/$name.job"
    ;;
  riek)
#   opt="-t intel-openmpi,gfortran-openmpi -c 'mpirun -bootstrap slurm -n NPROCPLUSONE'"
#   submit="sbatch -p $queue -J $name -N 9 -n $n --ntasks-per-node=4 -o $dir/$name.out $dir/$name.job"
    opt="-t intel-openmpi,gfortran-openmpi -c 'srun --mpi=pmi2 -n NPROCPLUSONE'"
    submit="sbatch -p $queue -J $name -n NPROCPLUSONE -o $dir/$name.out $dir/$name.job"
    ;;
  *)
    opt="-t intel-openmpi,gfortran-openmpi -c 'mpirun --hostfile \$PBS_NODEFILE -x CYANALIB -np NPROCPLUSONE --mca btl tcp,self'"
    submit="qsub -q $queue -N $name -j oe -o $dir/$name.out -l nodes=NNODES $name.job"
    ;;
esac


# ------ Start options and submit command ------

nproc1=`expr $nproc + 1`
nnodes=`echo $nproc1 $ncpu |\
        awk '{ n=int($1/$2)
               if (n==0) print "1:ppn=" $1
               else if (n*$2==$1) print n ":ppn=" $2
               else print n ":ppn=" $2 "+1:ppn=" $1-n*$2 }'`

opt=`echo $opt | sed "s/NPROCPLUSONE/$nproc1/"`
submit=`echo $submit | sed -e "s/NPROCPLUSONE/$nproc1/" -e "s/NNODES/$nnodes/"`
#echo "opt=$opt"
#echo "submit=$submit"


# ------ Write job script ------

echo "#!/bin/sh
cd `pwd`
date +'%d-%b-%Y %H:%M:%S'
t0=\`date +%s\`
uname -a
#head -1 /etc/issue
ulimit -Ss unlimited
grep '<< EOF$' $name.job | sed 's/ *<< EOF//'
$libdir/$prog -x $opt
echo nproc=$nproc
#printenv | fgrep PBS
#if [ \"\$PBS_NODEFILE\" ]; then echo PBS_NODEFILE:; cat \$PBS_NODEFILE; fi
" > $name.job
#case $cluster in
#  riek) echo "COMPILERVARS_ARCHITECTURE=intel64; export COMPILERVARS_ARCHITECTURE
#. /usr/local/intel2020/bin/compilervars.sh" >> $name.job;;
#esac
if [ "$passive" ]; then
  echo "echo \"$return\" > ../FROM" >> $name.job
fi
echo "\
$libdir/$prog $safe $opt << EOF
erract:=abort
nproc=$nproc
$macro $*
quit
EOF
echo
t1=\`date +%s\`; t1=\`expr \$t1 - \$t0\`
echo \"Total elapsed time: \$t1 s\"
date +'%d-%b-%Y %H:%M:%S'
" >> $name.job
if [ "$passive" ]; then
  echo "mv ../FROM ../DONE" >> $name.job
elif [ "$return" ]; then
  dir=`pwd`
  echo "ssh -f $host \"`dirname $dir`/returndata &\"" >> $name.job
fi
chmod +x $name.job


# ------ Submit job ------

submitjob() {
#echo "$submit"
eval $submit
case "$queue" in
-) echo "Background job $name started on $nproc processors.";;
*) echo "Job $name submitted to queue $queue, $nproc processors.";;
esac
}

rm -f $name.out
if [ "$after" ]; then
  echo "Job $name will start when file $after exists."
  (while [ ! -f $after ]; do sleep 10; done; submitjob) &
else
  submitjob
fi
