[vesta]

# last-tested-on: 2013-02-11
# last-tested-by: Erik Schnetter <schnetter@gmail.com>

# Machine description
nickname        = vesta
name            = Vesta
location        = ALCF
description     = Blue Gene/Q
webpage         = http://www.alcf.anl.gov/user-guides/mira-cetus-vesta
status          = experimental

# Access to this machine
hostname        = vesta.alcf.anl.gov
rsynccmd        = /home/eschnett/rsync-3.0.9/bin/rsync
aliaspattern    = ^vestalac[0-9]

# Source tree management
sourcebasedir   = /home/@USER@
disabled-thorns = <<EOT
    CaCUDA/CaCUDALib
    CaCUDA/WaveToyCUDA
    Carlile/CL_BSSN
    Carlile/CL_BSSN_Helper
    CarpetExtra/Nirvana
        CarpetDev/CarpetIONirvana
    CarpetExtra/TestLoopControl
    DGFE/AdvectHRSC
    DGFE/CPPUtils
    DGFE/EOS_Simple
    DGFE/EulerHRSC
    DGFE/FDCore
    DGFE/Filter
    DGFE/GREuler
    DGFE/HRSCCore
    DGFE/SDGMGrid
    DGFE/WaveToyDGFE
        McLachlan_dgfe/CL_BSSN_DG_P5
        McLachlan_dgfe/CL_BSSN_DG_P5_Helper
        McLachlan_dgfe/ML_FOZ4_DG_P5
        McLachlan_dgfe/ML_FOZ4_DG_P5_Helper
        McLachlan_dgfe/ML_WaveToyDGFE
    ExternalLibraries/CGNS
    ExternalLibraries/curl
        LSUThorns/Twitter
    ExternalLibraries/flickcurl
        LSUThorns/Flickr
    ExternalLibraries/FFTW3
        PITTNullCode/SphericalHarmonicRecon
    ExternalLibraries/libjpeg
        CactusIO/IOJpeg
    ExternalLibraries/libxml2
    ExternalLibraries/OpenSSL
    LSUThorns/CPUID
    ExternalLibraries/git
    ExternalLibraries/HYPRE
    ExternalLibraries/LORENE
        EinsteinInitialData/Meudon_Bin_BH
        EinsteinInitialData/Meudon_Bin_NS
        EinsteinInitialData/Meudon_Mag_NS
    ExternalLibraries/PETSc
        CactusElliptic/EllPETSc
        TAT/TATPETSc
        LSUDevelopment/WaveToyNoGhostsPETSc
    McLachlan/ML_CCZ4
    McLachlan/ML_CCZ4_Helper
    McLachlan/ML_CCZ4_Test
    McLachlan/ML_WaveToyMetricS2
EOT
# git, HYPRE, and LORENE work fine, but the library for thorn
# Formaline is then too large (on Intrepid?)
optionlist      = vesta.cfg
submitscript    = vesta.sub
runscript       = vesta.run
make            = make -j16

# Simulation management
basedir         = /gpfs/vesta-fs0/projects/SXSMHD/@USER@/simulations
cpu             = PowerPC A2
cpufreq         = 1.6
flop/cycle      = 8
efficiency      = 0.01
max-num-smt     = 4
num-smt         = 4
ppn             = 16
max-num-threads = 64
num-threads     = 16
memory          = 16384
I1size          = 16384
I1linesize      = 64
I1assoc         = 4
I1cores         = 1
D1size          = 16384
D1linesize      = 64
D1assoc         = 8
D1cores         = 1
L2size          = 33554432
L2linesize      = 128
L2assoc         = 16
L2cores         = 16
nodes           = 2048
min-ppn         = 16
allocation      = NO_ALLOCATION
queue           = default       # [default, low, single]
maxwalltime     = 2:00:00
submit          = cd @RUNDIR@-active && qsub -A @ALLOCATION@ -q @QUEUE@ -t @WALLTIME@ --mode script @('@CHAINED_JOB_ID@' != '' ? '--dependencies @CHAINED_JOB_ID@' : '')@ -n @NODES@ -M @EMAIL@ -O @SIMULATION_NAME@ -o @RUNDIR@-active/@SIMULATION_NAME@.out -e @RUNDIR@-active/@SIMULATION_NAME@.err @SCRIPTFILE@
getstatus       = qstat @JOB_ID@
stop            = qdel @JOB_ID@
submitpattern   = (\d+)
statuspattern   = ^@JOB_ID@[. ]
queuedpattern   = " (maxrun_hold|queued)  "
runningpattern  = " (exiting|killing|running|starting)  "
holdingpattern  = " (dep_hold|user_hold)  "
exechost        = qstat -f @JOB_ID@
exechostpattern = exec_host = (\w+)/
stdout          = ssh @EXECHOST@ cat /var/spool/torque/spool/@JOB_ID@.qb2.OU
stderr          = ssh @EXECHOST@ cat /var/spool/torque/spool/@JOB_ID@.qb2.ER
stdout-follow   = ssh @EXECHOST@ tail -n 100 -f /var/spool/torque/spool/@JOB_ID@.qb2.OU /var/spool/torque/spool/@JOB_ID@.qb2.ER