#!/bin/sh

echo "Preparing:"
set -x                          # Output commands
set -e                          # Abort on errors

cd @RUNDIR@
#
#
export GMON_OUT_PREFIX=gmon.out

echo "Checking:"
pwd
hostname
date

echo "PBS nodes:"
cat ${PBS_NODEFILE}
cat ${PBS_NODEFILE} > SIMFACTORY/NODES

echo "Environment:"

export OMP_NUM_THREADS=@NUM_THREADS@

env > SIMFACTORY/ENVIRONMENT

echo "Starting:"
MPI_NODEFILE=mpi_nodefile
for node in $(uniq ${PBS_NODEFILE}); do
    for ((i=0; i<@(@PPN_USED@/@NUM_THREADS@)@; ++i)); do
        echo ${node}
    done
done > ${MPI_NODEFILE}
export CACTUS_STARTTIME=$(date +%s)
MPICHDIR=/usr/local/openmpi-1.2-intel
# "--mca btl self,sm,tcp -mca btl_tcp_exclude eth0,ib0" uses gigE instead of IB
time ${MPICHDIR}/bin/mpirun --mca btl self,sm,tcp -mca btl_tcp_exclude eth0,ib0 -np @NUM_PROCS@ -machinefile ${MPI_NODEFILE} @EXECUTABLE@ -L 3 @PARFILE@

echo "Stopping:"
date

echo "Done."