#!/bin/sh

echo "Preparing:"
set -x                          # Output commands
set -e                          # Abort on errors

cd @RUNDIR@
#
#
export GMON_OUT_PREFIX=gmon.out

echo "Checking:"
pwd
hostname
date

echo "Environment:"
export MPI_NODEFILE=mpi_nodefile
export OMP_NUM_THREADS=@NUM_THREADS@
env > SIMFACTORY/ENVIRONMENT

echo "PBS nodes:"
cat ${PE_HOSTFILE}
cat ${PE_HOSTFILE} > SIMFACTORY/NODES

cat > start-exe <<EOF
exec /bin/env OMP_NUM_THREADS=@NUM_THREADS@ @EXECUTABLE@ \$*
EOF
chmod a+x start-exe

echo "Starting:"
for node in $(awk '{ print $1; }' ${PE_HOSTFILE} | uniq); do
    for (( proc=0; $proc<@(@PPN_USED@/@NUM_THREADS@)@; proc=$proc+1)); do
        echo ${node}
    done
done > ${MPI_NODEFILE}

export CACTUS_STARTTIME=$(date +%s)
MPICHDIR=/opt/mpich/intel
time ${MPICHDIR}/bin/mpirun -np @PROCS@ -machinefile ${MPI_NODEFILE} ./start-exe -L 3 @PARFILE@

echo "Stopping:"
date
rm -rf ${SCRATCHDIR}

echo "Done."