#! /bin/bash

exec >@RUNDIR@/@SIMULATION_NAME@.out
exec 2>@RUNDIR@/@SIMULATION_NAME@.err

echo "Preparing:"
set -x                          # Output commands
set -e                          # Abort on errors

cd @RUNDIR@-active

echo "Checking:"
pwd
hostname
date
cat ${PBS_NODEFILE} > SIMFACTORY/NODES

echo "Environment:"
export GMON_OUT_PREFIX=gmon.out
export OMP_NUM_THREADS=@NUM_THREADS@

env | sort > SIMFACTORY/ENVIRONMENT

# See /usr/local/doc/batch_scripts/openmpi.pbs for an example

#export NP=`wc -l ${PBS_NODEFILE} | cut -d'/' -f1`

module load openmpi_gcc-1.4.2
export OMPI_MCA_btl_openib_flags=1
export OMPI_MCA_btl_mpi_leave_pinned=0
export OMPI_MCA_btl_openib_warn_default_gid_prefix=0

#mpirun -np ${NP} -hostfile ${PBS_NODEFILE} ./a.out


export CACTUS_STARTTIME=$(date +%s)

# -npernode @(@PPN_USED@/@NUM_THREADS@)@

mpirun -v -np @NUM_PROCS@ @EXECUTABLE@ -L 3 @PARFILE@

echo "Stopping:"
date

echo "Done."