#!/bin/sh

echo "Preparing:"
set -x                          # Output commands
set -e                          # Abort on errors

cd @RUNDIR@
#
#
export GMON_OUT_PREFIX=gmon.out

# NOTE: The call to ibrun below depends on environment settings; make
# sure these module choices correspond to the MPI selection in the
# options file
module unload openmpi
module unload mvapich
module unload mvapich-devel
module unload mvapich-ud
module unload mvapich2
module load openmpi

echo "Checking:"
pwd
hostname
date

echo "Environment:"
export OMP_NUM_THREADS=@NUM_THREADS@
env > SIMFACTORY/ENVIRONMENT

cat $PE_HOSTFILE > SIMFACTORY/NODES || true

echo "Starting:"
# NOTE: ibrun depends on environment settings; make sure this
# corresponds to the MPI selection in the options file
export CACTUS_STARTTIME=$(date +%s)
time /share/sge6.2/default/pe_scripts/ibrun /share/sge6.2/default/pe_scripts/tacc_affinity @EXECUTABLE@ -L 3 @PARFILE@
# This should work as well
#time mpiexec -machinefile NODES -np @NUM_PROCS@ tacc_affinity @EXECUTABLE@ -L 3 @PARFILE@

echo "Stopping:"
date

echo "Done."