#! /bin/bash

echo "Preparing:"
set -x                          # Output commands
set -e                          # Abort on errors

cd @RUNDIR@-active

echo "Checking:"
pwd
hostname
date
cat ${PBS_NODEFILE} > SIMFACTORY/NODES

MVAPICH2_VERSION=mvapich2-0.9.8p2patched-intel-ofed-1.2-dbg

echo "Environment:"
export GMON_OUT_PREFIX=gmon.out
export MPD_NODEFILE=SIMFACTORY/mpd_nodefile
export MPI_NODEFILE=SIMFACTORY/mpi_nodefile
export MV2_SRQ_SIZE=4000        # ???
export OMP_NUM_THREADS=@NUM_THREADS@
export MPICHDIR=/usr/local/${MVAPICH2_VERSION}
env | sort > SIMFACTORY/ENVIRONMENT

# make sure LD_LIBRARY_PATH includes MVAPICH2
eval `soft-dec sh add +${MVAPICH2_VERSION}`

# make sure .mpd.conf exists
if [ -z ${HOME}/.mpd.conf ]; then
	echo MPD_SECRETWORD=simfactory_run > ${HOME}/.mpd.conf
fi

echo "Starting:"
uniq ${PBS_NODEFILE} > ${MPD_NODEFILE}
${MPICHDIR}/bin/mpdboot -v -n @NODES@ -f ${MPD_NODEFILE}
${MPICHDIR}/bin/mpdtrace -l
for node in $(cat ${MPD_NODEFILE}); do
    for ((proc=0; proc<@(@PPN_USED@/@NUM_THREADS@)@; ++proc)); do
        echo ${node}
    done
done > ${MPI_NODEFILE}

export CACTUS_STARTTIME=$(date +%s)
if [ @RUNDEBUG@ -eq 0 ]; then
	time ${MPICHDIR}/bin/mpirun -machinefile ${MPI_NODEFILE} -np @NUM_PROCS@ @EXECUTABLE@ -L 3 @PARFILE@
else
	eval `soft-dec sh add +totalview`
	${MPICHDIR}/bin/mpiexec -tv -n @NUM_PROCS@ @EXECUTABLE@ -L 3 @PARFILE@
fi
echo "Stopping:"
${MPICHDIR}/bin/mpdallexit
date

echo "Done."