[bluewaters-pgi]

# last-tested-on: 2013-11-04
# last-tested-by: Erik Schnetter <schnetter@gmail.com>

# Machine description
nickname        = bluewaters
name            = Blue Waters
location        = NCSA
description     = A Cray XE6/XK7 at the NCSA
webpage         = https://bluewaters.ncsa.illinois.edu/user-guide
status          = experimental

# Access to this machine
hostname        = h2ologin.ncsa.illinois.edu
rsynccmd        = /u/sciteam/eschnett/rsync-3.0.9/bin/rsync
envsetup        = source /etc/profile && module unload PrgEnv-cray PrgEnv-gnu PrgEnv-intel PrgEnv-pathscale PrgEnv-pgi && module load PrgEnv-pgi && module swap pgi pgi/13.10.0 && module load acml/5.3.1 && module load atp && module load cray-hdf5/1.8.11 && module load cray-petsc/3.4.3.0 && module load cudatoolkit/5.5.20-1.0402.7700.8.1 && module load fftw/3.3.0.4 && module load gsl/1.15.1 && module load papi/5.2.0 && module load pmi
aliaspattern    = $^

# Source tree management
sourcebasedir   = /u/sciteam/@USER@
disabled-thorns = <<EOT
    CactusTest/TestLoop
    Carpet/CarpetInterp2
    CarpetExtra/TestLoopControl
    ExternalLibraries/curl
        LSUThorns/Twitter
    ExternalLibraries/flickcurl
        LSUThorns/Flickr
    ExternalLibraries/libxml2
    ExternalLibraries/LORENE
        EinsteinInitialData/Meudon_Bin_BH
        EinsteinInitialData/Meudon_Bin_NS
        EinsteinInitialData/Meudon_Mag_NS
    ExternalLibraries/PETSc
        CactusElliptic/EllPETSc
        TAT/TATPETSc
        LSUDevelopment/WaveToyNoGhostsPETSc
    # Thorn ML_CCZ4 takes >30 minutes to compile
    McLachlan/ML_CCZ4
        McLachlan/ML_CCZ4_Helper
        McLachlan/ML_CCZ4_Test
EOT
enabled-thorns = <<EOT
EOT
optionlist      = bluewaters-pgi.cfg
submitscript    = bluewaters.sub
runscript       = bluewaters.run
make            = make -j16

# Simulation management
basedir         = /scratch/sciteam/@USER@/simulations
cpu             = AMD Opteron(TM) Processor 6276
cpufreq         = 2.45   # 2.3 is guaranteed, 2.45 is average
flop/cycle      = 4
max-num-smt     = 1
num-smt         = 1
ppn             = 32
mpn             = 4
max-num-threads = 32
num-threads     = 8
memory          = 65536
I1size          = 65536
I1linesize      = 64
I1assoc         = 2
I1cores         = 1
D1size          = 16384
D1linesize      = 64
D1assoc         = 4
D1cores         = 1
L2size          = 2097152
L2linesize      = 64
L2assoc         = 16
L2cores         = 1
L3size          = 6291456
L3linesize      = 64
L3assoc         = 64
L3cores         = 4
nodes           = 25712
min-ppn         = 32
allocation      = NO_ALLOCATION
queue           = normal:xe     # or normal:xk
maxwalltime     = 24:00:00
submit          = qsub @SCRIPTFILE@
getstatus       = qstat @JOB_ID@
stop            = qdel @JOB_ID@
submitpattern   = (\d+[.]nid\d+)
statuspattern   = "^@JOB_ID@[. ].* [QRH] "
queuedpattern   = "^@JOB_ID@[. ].* Q "
runningpattern  = "^@JOB_ID@[. ].* R "
holdingpattern  = "^@JOB_ID@[. ].* H "
#scratchbasedir   = /lustre/scratch/@USER@
#exechost        = /opt/torque/2.5.7/bin/qstat -f @JOB_ID@
#exechostpattern = exec_host = (\w+)/
# TODO: use qpeek
#stdout          = cat @SIMULATION_NAME@.out
#stderr          = cat @SIMULATION_NAME@.err
#stdout-follow   = tail -n 100 -f @SIMULATION_NAME@.out @SIMULATION_NAME@.err