[bluewaters]

# last-tested-on: 2013-11-04
# last-tested-by: Erik Schnetter <schnetter@gmail.com>

# Machine description
nickname        = bluewaters
name            = Blue Waters
location        = NCSA
description     = A Cray XE6/XK7 at the NCSA
webpage         = https://bluewaters.ncsa.illinois.edu/user-guide
status          = experimental

# Access to this machine
hostname        = h2ologin.ncsa.illinois.edu
rsynccmd        = /u/sciteam/eschnett/rsync-3.0.9/bin/rsync
sshcmd          = gsissh
sshopts         =
localsshsetup   = <<EOT
mkdir -p @SOURCEDIR@/.globus &&
: >> @SOURCEDIR@/.globus/proxy-tfca-ncsa &&
chmod go-rwx @SOURCEDIR@/.globus/proxy-tfca-ncsa &&
export X509_USER_PROXY=@SOURCEDIR@/.globus/proxy-tfca-ncsa &&
: mkdir -p @SOURCEDIR@/.globus/certificates-tfca-ncsa &&
: export X509_CERT_DIR=@SOURCEDIR@/.globus/certificates-tfca-ncsa &&
{
    {
        grid-proxy-info -issuer -file @SOURCEDIR@/.globus/proxy-tfca-ncsa 2>/dev/null | grep "^/C=US/O=National Center for Supercomputing Applications/" >/dev/null 2>&1 &&
        test $(grid-proxy-info -timeleft -file @SOURCEDIR@/.globus/proxy-tfca-ncsa 2>/dev/null) -gt 0 2>/dev/null
    } || {
        { grid-proxy-destroy 2>/dev/null || true; } &&
        myproxy-logon -p 7512 -s tfca.ncsa.illinois.edu -T -l @USER@ -o @SOURCEDIR@/.globus/proxy-tfca-ncsa
    }
} &&
{ globus-update-certificate-dir >/dev/null 2>&1 || true; }
EOT

envsetup        = <<EOT
    source /etc/profile
    module unload PrgEnv-cray PrgEnv-gnu PrgEnv-intel PrgEnv-pathscale PrgEnv-pgi
    module load PrgEnv-gnu
    module load cudatoolkit/5.0.35
    module load acml/5.3.1
    module load atp
    module load boost/1.53.0
    module load cray-hdf5/1.8.11
    module load cray-petsc/3.4.3.0
    module load cudatoolkit/5.5.20-1.0402.7700.8.1
    module load fftw/3.3.0.4
    module load gsl/1.15.1
    module load papi/5.2.0
    # module load perftools/6.0.1
    module load pmi
EOT
aliaspattern    = ^h2ologin[1-4](\.ncsa\.illinois\.edu)?$

# Source tree management
sourcebasedir   = /u/sciteam/@USER@
disabled-thorns = <<EOT
    ExternalLibraries/curl
        LSUThorns/Twitter
    ExternalLibraries/flickcurl
        LSUThorns/Flickr
    ExternalLibraries/libxml2
    ExternalLibraries/LORENE
        EinsteinInitialData/Meudon_Bin_BH
        EinsteinInitialData/Meudon_Bin_NS
        EinsteinInitialData/Meudon_Mag_NS
    ExternalLibraries/PETSc
        CactusElliptic/EllPETSc
        TAT/TATPETSc
EOT
enabled-thorns = <<EOT
    CactusExamples/HelloWorldCUDA
    ExternalLibraries/OpenCL
        CactusExamples/HelloWorldOpenCL
        CactusExamples/WaveToyOpenCL
        CactusUtils/OpenCLRunTime
        CactusUtils/Accelerator
        McLachlan/ML_BSSN_CL
        McLachlan/ML_BSSN_CL_Helper
        McLachlan/ML_WaveToy_CL
EOT
optionlist      = bluewaters-gnu.cfg
submitscript    = bluewaters.sub
runscript       = bluewaters.run
make            = make -j16

# Simulation management
basedir         = /scratch/sciteam/@USER@/simulations
cpu             = AMD Opteron(TM) Processor 6276
cpufreq         = 2.45   # 2.3 is guaranteed, 2.45 is average
flop/cycle      = 4
max-num-smt     = 1
num-smt         = 1
ppn             = 32
mpn             = 4
max-num-threads = 32
num-threads     = 8
memory          = 65536
I1size          = 65536
I1linesize      = 64
I1assoc         = 2
I1cores         = 1
D1size          = 16384
D1linesize      = 64
D1assoc         = 4
D1cores         = 1
L2size          = 2097152
L2linesize      = 64
L2assoc         = 16
L2cores         = 1
L3size          = 6291456
L3linesize      = 64
L3assoc         = 64
L3cores         = 4
nodes           = 25712
min-ppn         = 32
allocation      = NO_ALLOCATION
queue           = normal:xe     # or normal:xk
maxwalltime     = 24:00:00
submit          = qsub @SCRIPTFILE@
getstatus       = qstat @JOB_ID@
stop            = qdel @JOB_ID@
submitpattern   = (\d+[.]nid\d+)
statuspattern   = "^@JOB_ID@[. ].* [QRH] "
queuedpattern   = "^@JOB_ID@[. ].* Q "
runningpattern  = "^@JOB_ID@[. ].* R "
holdingpattern  = "^@JOB_ID@[. ].* H "
#scratchbasedir   = /lustre/scratch/@USER@
#exechost        = /opt/torque/2.5.7/bin/qstat -f @JOB_ID@
#exechostpattern = exec_host = (\w+)/
# TODO: use qpeek
#stdout          = cat @SIMULATION_NAME@.out
#stderr          = cat @SIMULATION_NAME@.err
#stdout-follow   = tail -n 100 -f @SIMULATION_NAME@.out @SIMULATION_NAME@.err