[surveyor]

# last-tested-on: 2013-11-04
# last-tested-by: Erik Schnetter <schnetter@gmail.com>
# PROBLEM: job submission fails

# Machine description
nickname        = surveyor
name            = Surveyor
location        = ALCF
description     = Blue Gene/P
webpage         = http://www.alcf.anl.gov/surveyor
status          = experimental

# Access to this machine
hostname        = surveyor.alcf.anl.gov
rsynccmd        = /home/eschnett/rsync-3.0.9/bin/rsync
aliaspattern    = ^login[0-9]\.surveyor

# Source tree management
sourcebasedir   = /home/@USER@
disabled-thorns = <<EOT
    CactusUtils/MemSpeed
    CaCUDA/CaCUDALib
    CaCUDA/WaveToyCUDA
    DGFE/AdvectHRSC
    DGFE/CPPUtils
    DGFE/EOS_Simple
    DGFE/EulerHRSC
    DGFE/FDCore
    DGFE/Filter
    DGFE/GREuler
    DGFE/HRSCCore
    DGFE/SDGMGrid
    DGFE/WaveToyDGFE
        McLachlan_dgfe/CL_BSSN_DG_P5
        McLachlan_dgfe/CL_BSSN_DG_P5_Helper
        McLachlan_dgfe/ML_FOZ4_DG_P5
        McLachlan_dgfe/ML_FOZ4_DG_P5_Helper
        McLachlan_dgfe/ML_WaveToyDGFE
    ExternalLibraries/CGNS
    ExternalLibraries/curl
        LSUThorns/Twitter
    ExternalLibraries/flickcurl
        LSUThorns/Flickr
    ExternalLibraries/FFTW3
        PITTNullCode/SphericalHarmonicRecon
    ExternalLibraries/libjpeg
        CactusIO/IOJpeg
    ExternalLibraries/libxml2
    ExternalLibraries/OpenSSL
    LSUThorns/CPUID
    ExternalLibraries/git
    ExternalLibraries/HYPRE
    ExternalLibraries/hwloc
    ExternalLibraries/LORENE
        EinsteinInitialData/Meudon_Bin_BH
        EinsteinInitialData/Meudon_Bin_NS
        EinsteinInitialData/Meudon_Mag_NS
EOT
# git, HYPRE, and LORENE work fine, but the library for thorn
# Formaline is then too large
optionlist      = intrepid-xlc.cfg
submitscript    = intrepid-xlc.sub
runscript       = intrepid-xlc.run
#make            = mkdir -p $HOME/tmp && env TMPDIR=$HOME/tmp make -j4
make            = make -j4

# Simulation management
basedir         = /pvfs-surveyor/@USER@/simulations
cpu             = PowerPC 450d
cpufreq         = 0.85
flop/cycle      = 4
ppn             = 4
max-num-threads = 4
num-threads     = 4
memory          = 2048
nodes           = 1024
min-ppn         = 4
allocation      = NO_ALLOCATION
queue           = default
maxwalltime     = 1:00:00
maxqueueslots   = 20
# --proccount @NUM_PROCS@
# --machine=@MACHINE@
submit          = cd @SOURCEDIR@ && @SIMFACTORY@ run @SIMULATION_NAME@ --machine=surveyor --restart-id=@RESTART_ID@ @FROM_RESTART_COMMAND@ && qsub -A @ALLOCATION@ -q @QUEUE@ -t @WALLTIME@ --mode @(["vn", "dual", "ERROR", "smp"][@NUM_THREADS@-1])@ -n @NODES@ @('@CHAINED_JOB_ID@' != '' ? '--dependencies @CHAINED_JOB_ID@' : '')@ -M @EMAIL@ -O @SIMULATION_NAME@ -o @RUNDIR@-active/@SIMULATION_NAME@.out -e @RUNDIR@-active/@SIMULATION_NAME@.err --cwd @RUNDIR@-active --env=OMP_NUM_THREADS=@NUM_THREADS@:BG_MAPPING=TXYZ:MPIRUN_LABEL=0:USER=@USER@ @EXECUTABLE@ -L 3 @PARFILE@
getstatus       = qstat @JOB_ID@
stop            = qdel @JOB_ID@
submitpattern   = (\d+)
statuspattern   = ^@JOB_ID@[. ]
queuedpattern   = " queued   "
runningpattern  = " running  "
holdingpattern  = " user hold  "
exechost        = qstat -f @JOB_ID@
exechostpattern = exec_host = (\w+)/
stdout          = ssh @EXECHOST@ cat /var/spool/torque/spool/@JOB_ID@.qb2.OU
stderr          = ssh @EXECHOST@ cat /var/spool/torque/spool/@JOB_ID@.qb2.ER
stdout-follow   = ssh @EXECHOST@ tail -n 100 -f /var/spool/torque/spool/@JOB_ID@.qb2.OU /var/spool/torque/spool/@JOB_ID@.qb2.ER