#! /bin/bash

echo "Preparing:"
set -x                          # Output commands
set -e                          # Abort on errors

cd @RUNDIR@-active

echo "Checking:"
pwd
hostname
date
cat ${PBS_NODES} > SIMFACTORY/NODES

echo "Environment:"
export GMON_OUT_PREFIX=gmon.out
export OMP_NUM_THREADS=@NUM_THREADS@
export MPICH_PTL_UNEX_EVENTS=204800
export CACTUS_STARTTIME=$(date +%s)

# NOTE: Hopper II has 2 sockets per node with 12 cores each
# aprun options:
#    -n    Number of MPI tasks
#    -N	   (Optional) Number of tasks per Hopper Node. Default is 24.
#    -d    (Optional) Depth, or number of threads, per MPI task. Use
#          this very important option in addition to OMP_NUM_THREADS
#          for OpenMP. Values can be 1-24. The default is 1. For
#          OpenMP values of 2-6 are recommended.
#    -S    (Optional) Number of tasks per NUMA node. Values can be
#          1-6; default 6
#    -sn   (Optional) Number of NUMA nodes to use per Hopper node.
#          Values can be 1-4; default 4
#    -ss   (Optional) Demands strict memory containment per NUMA node.
#          The default is the opposite - to allow remote NUMA node
#          memory access.
#    -cc   (Optional) Controls how tasks are bound to cores and NUMA
#          nodes. The recommend setting for most codes is -cc cpu
#          which restricts each task to run on a specific core.
#    -m    (Optional) Memory required per task. Three ways to use it:
#           -m size to request size bytes of memory; -m sizeh to
#           request huge pages of size size; and -m sizehs to require
#           huge pages of size size. See the large pages section below
#           for more details.

# Specify the number of MPI processes per node explicitly. Also
# specify the number of MPI processes per socket if this makes sense.
export NODE_PROCS='-N @(@PPN_USED@ / @NUM_THREADS@)@'
#export SOCKET_PROCS='@(@PPN_USED@ % (2 * @NUM_THREADS@) == 0 ? "-S @(@PPN_USED@ / (2 * @NUM_THREADS@))@" : "")@'

env | sort > SIMFACTORY/ENVIRONMENT

echo "Starting:"
aprun -n @NUM_PROCS@ -d @NUM_THREADS@ ${NODE_PROCS} ${SOCKET_PROCS} @EXECUTABLE@ -L 3 @PARFILE@

echo "Stopping:"
date

echo "Done."