[stampede]

# last-tested-on: 2014-10-21
# last-tested-by: Erik Schnetter <schnetter@gmail.com>

# NOTE: This machine configuration uses only the regular CPUs of
# Stampede, and ignores the MICs.

# Machine description
nickname        = stampede
name            = Stampede
location        = TACC
description     = A very large Linux cluster at TACC
webpage         = http://www.tacc.utexas.edu/user-services/user-guides/stampede-user-guide
status          = experimental

# Access to this machine
hostname        = stampede.tacc.utexas.edu
rsynccmd        = /home1/00507/eschnett/rsync-3.0.9/bin/rsync
envsetup        = <<EOT
    module load intel/13.1.1.163
    module load petsc/3.5
    module load papi
EOT
aliaspattern    = ^login[1234](\.stampede\.tacc\.utexas\.edu)?$

# Source tree management
# $HOME is not large enough
#sourcebasedir   = /home1/00507/@USER@
sourcebasedir   = /work/00507/@USER@
disabled-thorns = <<EOT
    ExternalLibraries/CGNS
    ExternalLibraries/curl
        LSUThorns/Twitter
    ExternalLibraries/flickcurl
        LSUThorns/Flickr
    ExternalLibraries/libxml2
    ExternalLibraries/Nirvana
        CarpetDev/CarpetIONirvana
        CarpetExtra/Nirvana
    ExternalLibraries/OpenSSL
EOT
enabled-thorns = <<EOT
    ExternalLibraries/OpenCL
        CactusExamples/HelloWorldOpenCL
        CactusExamples/WaveToyOpenCL
        CactusUtils/OpenCLRunTime
        CactusUtils/Accelerator
        McLachlan/ML_BSSN_CL
        McLachlan/ML_BSSN_CL_Helper
        McLachlan/ML_WaveToy_CL
    ExternalLibraries/PETSc
        CactusElliptic/EllPETSc
        CactusElliptic/TATelliptic
        CactusElliptic/TATPETSc
EOT
optionlist      = stampede-mvapich2.cfg
submitscript    = stampede.sub
runscript       = stampede-mvapich2.run
make            = make -j8

# Simulation management
basedir         = /scratch/00507/@USER@/simulations
cpu             = Intel(R) Xeon(R) CPU E5-2680 0 @ 2.70GHz
cpufreq         = 2.7
flop/cycle      = 8
ppn             = 16
spn             = 2
mpn             = 2
max-num-threads = 16
num-threads     = 8
memory          = 32768
I1size          = 32768
I1linesize      = 64
I1assoc         = 8
I1cores         = 1
D1size          = 32768
D1linesize      = 64
D1assoc         = 8
D1cores         = 1
L2size          = 262144
L2linesize      = 64
L2assoc         = 8
L2cores         = 1
L3size          = 20971520
L3linesize      = 64
L3assoc         = 20
L3cores         = 8
nodes           = 6400
min-ppn         = 16
allocation      = NO_ALLOCATION
queue           = normal        # [normal, large, development]
maxwalltime     = 48:00:00      # development has 4:0:0
maxqueueslots   = 49
submit          = sbatch @SCRIPTFILE@; sleep 60
getstatus       = squeue -j @JOB_ID@
stop            = scancel @JOB_ID@
submitpattern   = Submitted batch job ([0-9]+)
statuspattern   = '@JOB_ID@ '
queuedpattern   = ' PD '
runningpattern  = ' (CF|CG|R|TO) '
holdingpattern  = ' S '
#exechost        = head -n 1 SIMFACTORY/NODES
#exechostpattern = ^(\S+)
#stdout          = cat @SIMULATION_NAME@.out
#stderr          = cat @SIMULATION_NAME@.err
#stdout-follow   = tail -n 100 -f @SIMULATION_NAME@.out @SIMULATION_NAME@.err

# Intel MPI:
#
# Create a configuration using
#
#  --optionlist stampede-impi.cfg --runscript stampede-impi.run
#
# if you want to use Intel MPI, or set
#
#  optionlist      = stampede-impi.cfg
#  runscript       = stampede-impi.run
# 
# in your defs.local.ini for this to be the default for all configurations.