#!/bin/sh

echo "Preparing:"
set -x                          # Output commands
set -e                          # Abort on errors

cd @RUNDIR@
#
#
export GMON_OUT_PREFIX=gmon.out

echo "Checking:"
pwd
hostname
date

cat > aslimit.c <<EOF
#include <stdio.h>
#include <stdlib.h>
#include <sys/resource.h>
#include <unistd.h>
int main (int argc, char *argv[])
{
  struct rlimit rl;
  int ierr;
  if (argc<3) {
    printf("Usage: %s <memory limit in MByte> <command> ...\n", argv[0]);
    exit(1);
  }
  ierr=getrlimit(RLIMIT_AS, &rl);
  if (ierr) {
    fprintf(stderr, "Could not get AS limit\n");
    perror("getrlimit");
    exit(2);
  }
  /* set soft limit, keep hard limit */
  rl.rlim_cur=atoi(argv[1])*1024*1024;
  ierr=setrlimit(RLIMIT_AS, &rl);
  if (ierr) {
    fprintf(stderr, "Could not set AS limit\n");
    perror("setrlimit");
    exit(3);
  }
  ierr=execv(argv[2],&argv[2]);
  if (ierr) {
    fprintf(stderr, "Could not execv\n");
    perror("execv");
    exit(4);
  }
  return 0;
}
EOF
cc -o aslimit aslimit.c



echo "PBS nodes:"
cat ${PBS_NODEFILE}
cat ${PBS_NODEFILE} > SIMFACTORY/NODES

echo "Environment:"

export OMP_NUM_THREADS=@NUM_THREADS@

# Allow oversubscription of memory
MEMORY_PER_PROCESS=@(int (2.0 * @MEMORY@ * @NUM_THREADS@ / @PPN_USED@) - 1)@

export MPI_NODEFILE=mpi_nodefile

env > SIMFACTORY/ENVIRONMENT

echo "Starting:"

cat > startjob <<EOF
cd @RUNDIR@
$(${HOME}/dmalloc/bin/dmalloc debug3)
export OMP_NUM_THREADS=${OMP_NUM_THREADS}
exec ./aslimit ${MEMORY_PER_PROCESS} @EXECUTABLE@ \$*
EOF
chmod a+x startjob

for node in $(uniq ${PBS_NODEFILE}); do
    for (( proc=0; $proc<@(@PPN_USED@/@NUM_THREADS@)@; proc=$proc+1)); do
        echo ${node}
    done
done > ${MPI_NODEFILE}

export CACTUS_STARTTIME=$(date +%s)
MPICHDIR=/usr/local/packages/mpich-1.2.7p1-Intel-fc-9.1+gcc-3.4.6
time ${MPICHDIR}/bin/mpirun -np @PROCS@ -machinefile ${MPI_NODEFILE} @EXECUTABLE@ -L 3 @PARFILE@

echo "Stopping:"
date

rm -rf ${SCRATCHDIR}

echo "Done."