#! /bin/bash

echo "Preparing:"
set -x                          # Output commands
set -e                          # Abort on errors

cd @RUNDIR@-active

# NOTE: The call to ibrun below depends on environment settings; make
# sure these module choices correspond to the MPI selection in the
# options file
module unload openmpi
module unload mvapich
module unload mvapich-devel
module unload mvapich-ud
module unload mvapich2
module load mvapich2/1.2

module list

echo "Checking:"
pwd
hostname
date
cat "$PE_HOSTFILE" > SIMFACTORY/NODES || true

echo "Environment:"
export GMON_OUT_PREFIX=gmon.out
export OMP_NUM_THREADS=@NUM_THREADS@
env | sort > SIMFACTORY/ENVIRONMENT

echo "Starting:"
if test @PROCS_REQUESTED@ -gt 4000; then
    cache_binary $PWD @EXECUTABLE@
fi
# NOTE: ibrun depends on environment settings; make sure this
# corresponds to the MPI selection in the options file
export CACTUS_STARTTIME=$(date +%s)
time /share/sge6.2/default/pe_scripts/ibrun /share/sge6.2/default/pe_scripts/tacc_affinity @EXECUTABLE@ -L 3 @PARFILE@
# This should work as well
#time mpiexec -machinefile NODES -np @NUM_PROCS@ tacc_affinity @EXECUTABLE@ -L 3 @PARFILE@

echo "Stopping:"
date

echo "Done."