#!/bin/sh

# Redirect stdout and stderr
exec > @SIMULATION_NAME@.out 2> @SIMULATION_NAME@.err

echo "Preparing:"
set -x                          # Output commands
set -e                          # Abort on errors

cd @RUNDIR@
#
#
export GMON_OUT_PREFIX=gmon.out

echo "Checking:"
pwd
hostname
date

echo "Environment:"
export OMP_NUM_THREADS=@NUM_THREADS@
env > SIMFACTORY/ENVIRONMENT

echo "Nodes:"
echo 'numrel02' > SIMFACTORY/NODES

cat > SIMFACTORY/start-exe <<'EOF'
use strict;
my $simulation = fork;
if ($simulation == 0) {
    exec ('/bin/nice', '-n', '19', '/bin/env', 'OMP_NUM_THREADS=@NUM_THREADS@', '@EXECUTABLE@', @ARGV);
    die;
}
my $watchdog = fork;
if ($watchdog == 0) {
    sleep @WALLTIME_SECONDS@;
    kill 15, $simulation;
    sleep 60;
    kill 9, $simulation;
    exit;
}
wait;
kill 15, $watchdog;
EOF
chmod a+x SIMFACTORY/start-exe

echo "Starting:"
echo localhost > localhost
MPICH_DIR=/usr/local/packages/numrel/mpich-1.2.7p1
${MPICH_DIR}/bin/mpirun -np @NUM_PROCS@ -machinefile localhost SIMFACTORY/start-exe -L 3 @PARFILE@

echo "Stopping:"
date

mail -s 'Simulation @SIMULATION_NAME@ finished' @EMAIL@ <<EOF
The simulation below finished:

Name:        @SIMULATION_NAME@
ID:          @SIMULATION_ID@
Hostname:    @HOSTNAME@
Directory:   @RUNDIR@
Parfile:     @PARFILE@
Processes:   @PROCS@
PPN:         @PPN@
PPN used:    @PPN_USED@
Num threads: @NUM_THREADS@
EOF

rm -rf ${SCRATCHDIR}

echo "Done."