#!/bin/sh

echo "Preparing:"
set -x                          # Output commands
set -e                          # Abort on errors

cd @RUNDIR@
#
export GMON_OUT_PREFIX=gmon.out

echo "Checking:"
pwd
hostname
date

cat > aslimit.c <<EOF
#include <stdio.h>
#include <stdlib.h>
#include <sys/resource.h>
#include <unistd.h>
int main (int argc, char *argv[])
{
  struct rlimit rl;
  int ierr;
  if (argc<3) {
    printf("Usage: %s <memory limit in MByte> <command> ...\n", argv[0]);
    exit(1);
  }
  ierr=getrlimit(RLIMIT_AS, &rl);
  if (ierr) {
    fprintf(stderr, "Could not get AS limit\n");
    perror("getrlimit");
    exit(2);
  }
  /* set soft limit, keep hard limit */
  rl.rlim_cur=atoi(argv[1])*1024*1024;
  ierr=setrlimit(RLIMIT_AS, &rl);
  if (ierr) {
    fprintf(stderr, "Could not set AS limit\n");
    perror("setrlimit");
    exit(3);
  }
  ierr=execv(argv[2],&argv[2]);
  if (ierr) {
    fprintf(stderr, "Could not execv\n");
    perror("execv");
    exit(4);
  }
  return 0;
}
EOF
cc -o aslimit aslimit.c

echo "PBS nodes:"
cat ${PBS_NODEFILE}
cat ${PBS_NODEFILE} > SIMFACTORY/NODES

echo "Environment:"

export OMP_NUM_THREADS=@NUM_THREADS@

# Allow oversubscription of memory
MEMORY_PER_PROCESS=@(int (2.0 * @MEMORY@ * @NUM_THREADS@ / @PPN_USED@) - 1)@

env > SIMFACTORY/ENVIRONMENT

echo "Starting:"

export CACTUS_STARTTIME=$(date +%s)
MPICHDIR=/usr/local/packages/openmpi-1.2.4-intel10.0
#time ${MPICHDIR}/bin/mpirun -np @PROCS@ --bynode /bin/env OMP_NUM_THREADS=${OMP_NUM_THREADS} ./aslimit ${MEMORY_PER_PROCESS} @EXECUTABLE@ -L 3 @PARFILE@
time ${MPICHDIR}/bin/mpirun -np @PROCS@ --bynode /bin/env OMP_NUM_THREADS=${OMP_NUM_THREADS} @EXECUTABLE@ -L 3 @PARFILE@

echo "Stopping:"
date

rm -rf ${SCRATCHDIR}

echo "Done."