#! /bin/bash

echo "Preparing:"
set -x                          # Output commands
set -e                          # Abort on errors

cd @RUNDIR@-active

echo "Checking:"
pwd
hostname
date
echo "${LSB_MCPU_HOSTS}" > SIMFACTORY/NODES

echo "Environment:"
# One of these variables confuses mpirun, so we unset them all
unset LSB_ACCT_FILE
unset LSB_BATCH_JID
unset LSB_CHKFILENAME
unset LSB_EEXEC_REAL_GID
unset LSB_EEXEC_REAL_UID
unset LSB_ERRORFILE
unset LSB_EXIT_PRE_ABORT
unset LSB_EXIT_REQUEUE
unset LSB_HOSTS
unset LSB_JOB_EXECUSER
unset LSB_JOBEXIT_STAT
unset LSB_JOBFILENAME
unset LSB_JOBID
unset LSB_JOBINDEX
unset LSB_JOBNAME
unset LSB_JOBRES_PID
unset LSB_MCPU_HOSTS
unset LSB_OUTPUTFILE
unset LSB_QUEUE
unset LSB_SLURM_ERRORFILE
unset LSB_SLURM_OUTPUTFILE
unset LSB_SUB_HOST
unset LSB_TRAPSIGS
unset LSB_UNIXGROUP
unset SLURM_CPU_BIND_LIST
unset SLURM_CPU_BIND_TYPE
unset SLURM_CPU_BIND_VERBOSE
unset SLURM_JOBID
unset SLURM_LOCALID
unset SLURM_NNODES
unset SLURM_NODEID
unset SLURM_NODELIST
unset SLURM_NPROCS
unset SLURM_PROCID
unset SLURM_STEPID
unset SLURM_TASK_PID
unset SLURM_TASKS_PER_NODE
unset SLURM_UMASK
export MPI_HOSTFILE=mpi_hostfile
export MPI_HOSTS=mpi_hosts
export OMP_NUM_THREADS=@NUM_THREADS@
env | sort > SIMFACTORY/ENVIRONMENT

echo "Starting:"
for host in ${LSB_HOSTS}; do
    echo ${host}
done > ${MPI_HOSTS}
for node in $(uniq ${MPI_HOSTS}); do
    for ((proc=0; $proc<@(@PPN_USED@/@NUM_THREADS@)@; proc=$proc+1)); do
        echo ${node}
    done
done > ${MPI_HOSTFILE}
export CACTUS_STARTTIME=$(date +%s)
time /opt/sharcnet/openmpi/1.1.1/bin/mpirun -np @NUM_PROCS@ -hostfile ${MPI_HOSTFILE} -bynode @EXECUTABLE@ -L 3 @PARFILE@

echo "Stopping:"
date

echo "Done."