#! /bin/bash

echo "Preparing:"
set -x                          # Output commands
set -e                          # Abort on errors

cd @RUNDIR@-active

echo "Checking:"
pwd
hostname
date
cat ${PBS_NODEFILE} > SIMFACTORY/NODES

export LD_LIBRARY_PATH=/ichec/packages/intel/composerxe_cc/2011.7.256/composer_xe_2011_sp1.7.256/compiler/lib/intel64:$LD_LIBRARY_PATH

echo "Environment:"
export CACTUS_NUM_PROCS=@NUM_PROCS@
export CACTUS_NUM_THREADS=@NUM_THREADS@
export GMON_OUT_PREFIX=gmon.out
export OMP_NUM_THREADS=@NUM_THREADS@
export MKL_NUM_THREADS=1
export MPICHDIR=/ichec/packages/mvapich/1.5.1p1-intel
export MALLOC_MMAP_MAX_=0
export MALLOC_TRIM_THRESHOLD_=-1
export MPD_NODEFILE=mpd_nodefile
export MPI_NODEFILE=mpi_nodefile
if [ $OMP_NUM_THREADS -eq 12 ]; then
    export MV2_CPU_MAPPING="0,1,2,3,4,5,6,7,8,9,10,11"
elif [ $OMP_NUM_THREADS -eq 6 ]; then
    export MV2_CPU_MAPPING="0,1,2,3,4,5:6,7,8,9,10,11"
elif [ $OMP_NUM_THREADS -eq 3 ]; then
    export MV2_CPU_MAPPING="0,1,2:3,4,5:6,7,8:9,10,11"
elif [ $OMP_NUM_THREADS -eq 2 ]; then
    export MV2_CPU_MAPPING="0,1:2,3:4,5:6,7:8,9:10,11"
elif [ $OMP_NUM_THREADS -eq 1 ]; then
    export MV2_CPU_MAPPING="0:1:2:3:4:5:6:7:8:9:10:11"
fi
env | sort > SIMFACTORY/ENVIRONMENT

echo "Starting:"
uniq ${PBS_NODEFILE} > ${MPD_NODEFILE}
for node in $(cat ${MPD_NODEFILE}); do
    for ((proc=0; $proc<@(@PPN_USED@/@NUM_THREADS@)@; proc=$proc+1)); do
        echo ${node}
    done
done > ${MPI_NODEFILE}
export CACTUS_STARTTIME=$(date +%s)

/usr/bin/time mpiexec -npernode @(@PPN_USED@/@NUM_THREADS@)@ @EXECUTABLE@ -L 3 @PARFILE@

echo "Stopping:"
date

echo "Done."