[bluewaters]

# last-tested-on: 2013-11-04
# last-tested-by: Erik Schnetter <schnetter@gmail.com>

# Machine description
nickname        = bluewaters
name            = Blue Waters
location        = NCSA
description     = A Cray XE6/XK7 at the NCSA
webpage         = https://bluewaters.ncsa.illinois.edu/user-guide
status          = experimental

# Access to this machine
hostname        = h2ologin.ncsa.illinois.edu
rsynccmd        = /u/sciteam/eschnett/rsync-3.0.9/bin/rsync
envsetup        = source /etc/profile && module unload PrgEnv-cray PrgEnv-gnu PrgEnv-intel PrgEnv-pathscale PrgEnv-pgi && module load PrgEnv-cray && module load atp && module swap cce cce/8.1.5 && module load fftw/3.3.0.1 && module load hdf5/1.8.8 && module load papi/5.0.1
aliaspattern    = $^

# Source tree management
sourcebasedir   = /u/sciteam/@USER@
disabled-thorns = <<EOT
    ExternalLibraries/curl
        LSUThorns/Twitter
    ExternalLibraries/flickcurl
        LSUThorns/Flickr
    ExternalLibraries/libxml2
    ExternalLibraries/LORENE
        EinsteinInitialData/Meudon_Bin_BH
        EinsteinInitialData/Meudon_Bin_NS
        EinsteinInitialData/Meudon_Mag_NS
    ExternalLibraries/PETSc
        CactusElliptic/EllPETSc
        TAT/TATPETSc
        LSUDevelopment/WaveToyNoGhostsPETSc
    # Cray:
    #CarpetExtra/TestLoopControl
    #DGFE/AdvectHRSC
    #DGFE/Filter
    #DGFE/HRSCCore
    #DGFE/SDGMGrid
    #DGFE/WaveToyDGFE
    #ExternalLibraris/HYPRE
    #ExternalLibraris/PAPI
    #ExternalLibraris/PETSc
    #ExternalLibraris/flickcurl
    #ExternalLibraris/libxml2
    #LSUThorns/CPUID
    #McLachlan_dgfe/CL_BSSN_DG_P5
    #McLachlan_dgfe/CL_BSSN_DG_P5_Helper
    #McLachlan_dgfe/ML_FOZ4_DG_P5
    #McLachlan_dgfe/ML_FOZ4_DG_P5_Helper
    #McLachlan_dgfe/ML_WaveToyDGFE
    #CactusExamples/WaveToyCUDA
    #CactusTest/TestFortranCrayPointers
    #CactusTest/TestLoop
    # PGI:
    #CactusTest/TestLoop
    #ExternalLibraris/CGNS
    #ExternalLibraris/HYPRE
    #ExternalLibraris/PAPI
    #ExternalLibraris/PETSc
    #ExternalLibraris/curl
    #ExternalLibraris/flickcurl
    #DGFE/*
    #Llama/Coordinates
    #McLachlan/ML_WaveToyMetricS2
EOT
enabled-thorns = <<EOT
#    CactusExamples/WaveToyCUDA
#    ExternalLibraries/OpenCL
#        CactusExamples/WaveToyOpenCL
#        CactusUtils/Accelerator
#        CactusUtils/OpenCLRunTime
#        McLachlan/ML_BSSN_CL
#        McLachlan/ML_BSSN_CL_Helper
#        McLachlan/ML_WaveToy_CL
EOT
#optionlist      = bluewaters-cray.cfg
optionlist      = bluewaters-gnu.cfg
#optionlist      = bluewaters-pgi.cfg
submitscript    = bluewaters.sub
runscript       = bluewaters.run
make            = make -j16

# Simulation management
basedir         = /scratch/sciteam/@USER@/simulations
cpu             = AMD Opteron(TM) Processor 6276
cpufreq         = 2.45   # 2.3 is guaranteed, 2.45 is average
flop/cycle      = 4
max-num-smt     = 1
num-smt         = 1
ppn             = 32
mpn             = 4
max-num-threads = 32
num-threads     = 8
memory          = 65536
I1size          = 65536
I1linesize      = 64
I1assoc         = 2
I1cores         = 1
D1size          = 16384
D1linesize      = 64
D1assoc         = 4
D1cores         = 1
L2size          = 2097152
L2linesize      = 64
L2assoc         = 16
L2cores         = 1
L3size          = 6291456
L3linesize      = 64
L3assoc         = 64
L3cores         = 4
nodes           = 25712
min-ppn         = 32
allocation      = NO_ALLOCATION
queue           = normal:xe     # or normal:xk
maxwalltime     = 24:00:00
submit          = qsub @SCRIPTFILE@
getstatus       = qstat @JOB_ID@
stop            = qdel @JOB_ID@
submitpattern   = (\d+[.]nid\d+)
statuspattern   = "^@JOB_ID@[. ].* [QRH] "
queuedpattern   = "^@JOB_ID@[. ].* Q "
runningpattern  = "^@JOB_ID@[. ].* R "
holdingpattern  = "^@JOB_ID@[. ].* H "
#scratchbasedir   = /lustre/scratch/@USER@
#exechost        = /opt/torque/2.5.7/bin/qstat -f @JOB_ID@
#exechostpattern = exec_host = (\w+)/
# TODO: use qpeek
#stdout          = cat @SIMULATION_NAME@.out
#stderr          = cat @SIMULATION_NAME@.err
#stdout-follow   = tail -n 100 -f @SIMULATION_NAME@.out @SIMULATION_NAME@.err