#! /bin/bash

echo "Preparing:"
set -x                          # Output commands
set -e                          # Abort on errors

cd @RUNDIR@-active

echo "Checking:"
pwd
hostname
date
cat ${PBS_NODES} > SIMFACTORY/NODES

echo "Environment:"
export GMON_OUT_PREFIX=gmon.out
export OMP_NUM_THREADS=@NUM_THREADS@
export MPICH_PTL_UNEX_EVENTS=204800
export CACTUS_STARTTIME=$(date +%s)

# NOTE: Hopper has 2 sockets per node with 6 cores each
# aprun options:
# These are necessary:
#    -n   Total number of MPI processes
#    -d   Specifies number of cores per MPI process (1-12)
# These are implied:
#    -N   Number of MPI processes per node (1-12)
#    -S   Number of MPI processes per socket (1-6)
# Memory options:
#    -m   Amount of memory per MPI process (with suffix k, M, or G)
#    -ss  Allocate memory only on same socket
# Specify the number of MPI processes per node explicitly. Also
# specify the number of MPI processes per socket if this makes sense.
export NODE_PROCS='-N @(@PPN_USED@ / @NUM_THREADS@)@'
export SOCKET_PROCS='@(@PPN_USED@ % (2 * @NUM_THREADS@) == 0 ? "-S @(@PPN_USED@ / (2 * @NUM_THREADS@))@" : "")@'

env | sort > SIMFACTORY/ENVIRONMENT

echo "Starting:"
#aprun -n @NUM_PROCS@ -N @(@PPN_USED@/@NUM_THREADS@)@ -d @NUM_THREADS@ @EXECUTABLE@ -L 3 @PARFILE@
aprun -n @NUM_PROCS@ -d @NUM_THREADS@ ${NODE_PROCS} ${SOCKET_PROCS} @EXECUTABLE@ -L 3 @PARFILE@

echo "Stopping:"
date

echo "Done."