#!/bin/sh

# Redirect stdout and stderr
exec > @SIMULATION_NAME@.out 2> @SIMULATION_NAME@.err

echo "Preparing:"
set -x                          # Output commands
set -e                          # Abort on errors

cd @RUNDIR@
#
#
export GMON_OUT_PREFIX=gmon.out

echo "Checking:"
pwd
hostname
date

echo "Environment:"
export OMP_NUM_THREADS=@NUM_THREADS@
env > SIMFACTORY/ENVIRONMENT

if [ @NUM_THREADS@ -ne 1 ]; then
    echo 'OpenMP is currently not supported on Wilson'
    exit 1
fi

echo "Nodes:"
echo 'numrel02' > SIMFACTORY/NODES

cat > SIMFACTORY/start-exe <<EOF
echo "[Starting simulation with PID $$]"
(
    # Start a watchdog that terminates the simulation after the wall
    # time limit has been reached
    sleep @WALLTIME_SECONDS@
    echo "[Wall time limit reached, stopping simulation with PID $$]"
    kill $$
    sleep 60
    kill -9 $$
    sleep 10
    if ps $pid > /dev/null 2>&1; then
        echo "[ERROR: Could not stop simulation with PID $$]"
        exit 1
    fi
    echo "[Simulation with PID $$ stopped]"
) &
exec /bin/env OMP_NUM_THREADS=@NUM_THREADS@ @EXECUTABLE@ \$*
EOF
chmod a+x SIMFACTORY/start-exe

echo "Starting:"
echo localhost > localhost
export CACTUS_STARTTIME=$(date +%s)
MPICH_DIR=/usr/local/packages/numrel/mpich-1.2.7p1
${MPICH_DIR}/bin/mpirun -np @NUM_PROCS@ -machinefile localhost SIMFACTORY/start-exe -L 3 @PARFILE@

echo "Stopping:"
date

mail -s 'Simulation @SIMULATION_NAME@ finished' @USER@@cct.lsu.edu <<EOF
The simulation below finished:

Name:        @SIMULATION_NAME@
ID:          @SIMULATION_ID@
Hostname:    @HOSTNAME@
Directory:   @RUNDIR@
Parfile:     @PARFILE@
Processes:   @PROCS@
PPN:         @PPN@
PPN used:    @PPN_USED@
Num threads: @NUM_THREADS@
EOF

rm -rf ${SCRATCHDIR}

echo "Done."