#! /bin/bash

echo "Preparing:"
set -x                          # Output commands
set -e                          # Abort on errors

cd @RUNDIR@-active

echo "Checking:"
pwd
hostname
date
cat ${MPI_NODEFILE} > SIMFACTORY/NODES

echo "Environment:"
export GMON_OUT_PREFIX=gmon.out
export OMP_NUM_THREADS=@NUM_THREADS@
export MPIDIR=/cluster/openmpi/1.4.1
export MPI_NODEFILE=${TMPDIR}/machines
env | sort > SIMFACTORY/ENVIRONMENT

echo "Starting:"
uniq ${MPI_NODEFILE} > PROC_NODES
for node in $(cat PROC_NODES); do
    for (( proc=0; $proc<@(@PPN_USED@/@NUM_THREADS@)@; proc=$proc+1)); do
        echo ${node}
    done
done > ${MPI_NODEFILE}

export CACTUS_STARTTIME=$(date +%s)
time ${MPIDIR}/bin/mpirun -v --mca btl openib,self --mca mpi_leave_pinned 0 -np @NUM_PROCS@ -npernode @(@PPN_USED@/@NUM_THREADS@)@ @EXECUTABLE@ -L 3 @PARFILE@

echo "Stopping:"
date

echo "Done."