#!/bin/sh

echo "Preparing:"
set -x                          # Output commands
set -e                          # Abort on errors

cd @RUNDIR@
#
#
export GMON_OUT_PREFIX=gmon.out

echo "Checking:"
pwd
hostname
date

cat ${PBS_NODEFILE} > SIMFACTORY/NODES

echo "Environment:"
export OMP_NUM_THREADS=@NUM_THREADS@

# for jobs with >128 MPI processes one may need to increase
# the unexpected receive queue buffer size
#export MPICH_UNEX_BUFFER_SIZE=40M
export MPICH_UNEX_BUFFER_SIZE=100M

# Setting this seems to make MPI hang
#export MPICH_MAX_SHORT_MSG_SIZE=16000

export MPICH_PTL_UNEX_EVENTS=204800

# NAMD uses:
#export MPICH_PTL_SEND_CREDITS=-1
#export MPICH_MAX_SHORT_MSG_SIZE=8000
#export MPICH_PTL_UNEX_EVENTS=80000
#export MPICH_UNEX_BUFFER_SIZE=100M

export MPICH_ENV_DISPLAY=1
export MPICH_VERSION_DISPLAY=1
export MPICH_ABORT_ON_ERROR=1
env > SIMFACTORY/ENVIRONMENT

cat > gdb.cmds <<EOF
run
thread apply all info registers
thread apply all info locals
thread apply all backtrace full
EOF

# rsync something (e.g. initial data files) to scratch for simulation to use
# before it starts
if [ "x$RSYNC_TO_SCRATCH" != "x" ]; then
  echo "Copying files in $RSYNC_TO_SCRATCH to @RUNDIR@"
  rsync -Pav "$RSYNC_TO_SCRATCH" @RUNDIR@
fi

echo "Starting:"
export CACTUS_STARTTIME=$(date +%s)

# NOTE: Kraken has 2 sockets per node with 2 cores each
# aprun options:
# These are necessary:
# -n	Total number of MPI processes
# -d	Specifies number of cores per MPI process (1-4)
# These are implied:
# -N	Number of MPI processes per node (1-4)
# -S	Number of MPI processes per socket (1-2)
aprun -n @NUM_PROCS@ -d @NUM_THREADS@ -ss @EXECUTABLE@ -L 3 @PARFILE@
#aprun -n @NUM_PROCS@ -d @NUM_THREADS@ -ss gdb -batch -x gdb.cmds --args @EXECUTABLE@ -L 3 @PARFILE@

echo "Stopping:"
date

echo "Done."