[bluewaters]

# last-tested-on: 2013-04-29
# last-tested-by: Erik Schnetter <schnetter@gmail.com>

# Machine description
nickname        = bluewaters
name            = Blue Waters
location        = NCSA
description     = A Cray XE6/XK7 at the NCSA
webpage         = https://bluewaters.ncsa.illinois.edu/user-guide
status          = experimental

# Access to this machine
hostname        = h2ologin.ncsa.illinois.edu
rsynccmd        = /u/sciteam/eschnett/rsync-3.0.9/bin/rsync
# module load perftools/6.0.1
#envsetup        = module unload PrgEnv-cray PrgEnv-gnu PrgEnv-intel PrgEnv-pathscale PrgEnv-pgi && module load PrgEnv-cray && module load acml/5.0.0 && module load fftw/3.3.0.1 && module load gsl/1.15 && module load hdf5/1.8.8 && module load petsc/3.3.00 && module load papi/5.0.1
envsetup        = source /etc/profile && module unload PrgEnv-cray PrgEnv-gnu PrgEnv-intel PrgEnv-pathscale PrgEnv-pgi && module load PrgEnv-gnu && module load cudatoolkit/5.0.35.102 && module load acml/5.0.0 && module load fftw/3.3.0.1 && module load gsl/1.15 && module load hdf5/1.8.8 && module load petsc/3.3.00 && module load papi/5.0.1 && module load pmi
#envsetup        = module unload PrgEnv-cray PrgEnv-gnu PrgEnv-intel PrgEnv-pathscale PrgEnv-pgi && module load PrgEnv-pgi && module load acml/5.0.0 && module load fftw/3.3.0.1 && module load gsl/1.15 && module load hdf5/1.8.8 && module load petsc/3.3.00 && module load papi/5.0.1
aliaspattern    = ^h2ologin[1-4](\.ncsa\.illinois\.edu)?$

# Source tree management
sourcebasedir   = /u/sciteam/@USER@
disabled-thorns = <<EOT
    ExternalLibraries/curl
        LSUThorns/Twitter
    ExternalLibraries/flickcurl
        LSUThorns/Flickr
    ExternalLibraries/libxml2
    ExternalLibraries/LORENE
        EinsteinInitialData/Meudon_Bin_BH
        EinsteinInitialData/Meudon_Bin_NS
        EinsteinInitialData/Meudon_Mag_NS
    ExternalLibraries/PETSc
        CactusElliptic/EllPETSc
        TAT/TATPETSc
        LSUDevelopment/WaveToyNoGhostsPETSc
    # PGI:
    #ExternalLibraris/CGNS
    #ExternalLibraris/HYPRE
    #DGFE/*
    #*/Coordinates
    #no enabled-thorns
EOT
enabled-thorns = <<EOT
    CactusExamples/WaveToyCUDA
    ExternalLibraries/OpenCL
        CactusExamples/WaveToyOpenCL
        CactusUtils/Accelerator
        CactusUtils/OpenCLRunTime
        McLachlan/ML_BSSN_CL
        McLachlan/ML_BSSN_CL_Helper
        McLachlan/ML_WaveToy_CL
EOT
#optionlist      = bluewaters-cray.cfg
optionlist      = bluewaters-gnu.cfg
#optionlist      = bluewaters-pgi.cfg
submitscript    = bluewaters.sub
runscript       = bluewaters.run
make            = make -j16

# Simulation management
basedir         = /scratch/sciteam/@USER@/simulations
cpu             = AMD Opteron(TM) Processor 6276
cpufreq         = 2.3
flop/cycle      = 8
max-num-smt     = 2
num-smt         = 2
ppn             = 16
mpn             = 4
max-num-threads = 32
num-threads     = 8
memory          = 65536
I1size          = 65536
I1linesize      = 64
I1assoc         = 2
I1cores         = 1
D1size          = 16384         # 16384x2
D1linesize      = 64
D1assoc         = 4
D1cores         = 1
L2size          = 2097152
L2linesize      = 64
L2assoc         = 16
L2cores         = 1
L3size          = 6291456
L3linesize      = 64
L3assoc         = 64
L3cores         = 4
nodes           = 25712
min-ppn         = 16
allocation      = NO_ALLOCATION
queue           = batch:xe      # or batch:xk
maxwalltime     = 12:00:00
submit          = qsub @SCRIPTFILE@
getstatus       = qstat @JOB_ID@
stop            = qdel @JOB_ID@
submitpattern   = (\d+[.]nid\d+)
statuspattern   = "^@JOB_ID@[. ].* [QRH] "
queuedpattern   = "^@JOB_ID@[. ].* Q "
runningpattern  = "^@JOB_ID@[. ].* R "
holdingpattern  = "^@JOB_ID@[. ].* H "
#scratchbasedir   = /lustre/scratch/@USER@
#exechost        = /opt/torque/2.5.7/bin/qstat -f @JOB_ID@
#exechostpattern = exec_host = (\w+)/
#stdout          = cat @SIMULATION_NAME@.out
#stderr          = cat @SIMULATION_NAME@.err
#stdout-follow   = tail -n 100 -f @SIMULATION_NAME@.out @SIMULATION_NAME@.err