[stampede-mic]

# last-tested-on: 2013-11-04
# last-tested-by: Erik Schnetter <schnetter@gmail.com>

# NOTE: This machine configuration uses only the MICs of Stampede.

# Machine description
nickname        = stampede-mic
name            = Stampede (MICs)
location        = TACC
description     = A very large Linux cluster at TACC
webpage         = http://www.tacc.utexas.edu/user-services/user-guides/stampede-user-guide
status          = experimental

# Access to this machine
hostname        = stampede.tacc.utexas.edu
rsynccmd        = /home1/00507/eschnett/rsync-3.0.9/bin/rsync
#? envsetup        = module load intel/13.1.1.163 && module unload mvapich2 && module load impi/4.1.1.036 && module load papi-mic
aliaspattern    = NO_MATCH #^login[1234]-(\.stampede\.tacc\.utexas\.edu)?$

# Source tree management
# $HOME is not large enough
#sourcebasedir   = /home1/00507/@USER@/mic
sourcebasedir   = /work/00507/@USER@/mic
disabled-thorns = <<EOT
    ExternalLibraries/CGNS
    ExternalLibraries/curl
        LSUThorns/Twitter
    ExternalLibraries/FFTW3
        PITTNullCode/SphericalHarmonicRecon
    ExternalLibraries/flickcurl
        LSUThorns/Flickr
    ExternalLibraries/HYPRE
    ExternalLibraries/libjpeg
        CactusIO/IOJpeg
    ExternalLibraries/libxml2
    ExternalLibraries/LORENE
        EinsteinInitialData/Meudon_Bin_BH
        EinsteinInitialData/Meudon_Bin_NS
        EinsteinInitialData/Meudon_Mag_NS
    ExternalLibraries/Nirvana
        CarpetDev/CarpetIONirvana
        CarpetExtra/Nirvana
    ExternalLibraries/OpenSSL
    ExternalLibraries/PAPI
    ExternalLibraries/pciutils
    ExternalLibraries/PETSc
        CactusElliptic/EllPETSc
        TAT/TATPETSc
EOT
optionlist      = stampede-mic.cfg
submitscript    = stampede-mic.sub
runscript       = stampede-mic.run
make            = make -j8

# Simulation management
basedir         = /scratch/00507/@USER@/simulations
cpu             = Intel Xeon Phi SE10P Coprocessor
cpufreq         = 1.1
flop/cycle      = 16
efficiency      = 0.01
max-num-smt     = 4
num-smt         = 4
ppn             = 60            # could be 61
spn             = 1
mpn             = 1
max-num-threads = 240           # could be 244
num-threads     = 60            # could be 61
memory          = 7692
I1size          = 32768
I1linesize      = 64
I1assoc         = 8
I1cores         = 1
D1size          = 32768
D1linesize      = 64
D1assoc         = 8
D1cores         = 1
L2size          = 524288
L2linesize      = 64
L2assoc         = 8
L2cores         = 1
nodes           = 6400          # 1
min-ppn         = 1
allocation      = NO_ALLOCATION
queue           = normal-mic
maxwalltime     = 24:00:00
maxqueueslots   = 49
submit          = sbatch @SCRIPTFILE@
getstatus       = squeue -j @JOB_ID@
stop            = scancel @JOB_ID@
submitpattern   = Submitted batch job ([0-9]+)
statuspattern   = ' @JOB_ID@ '
queuedpattern   = ' PD '
runningpattern  = ' R '
holdingpattern  = ' S '
#exechost        = head -n 1 SIMFACTORY/NODES
#exechostpattern = ^(\S+)
#stdout          = cat @SIMULATION_NAME@.out
#stderr          = cat @SIMULATION_NAME@.err
#stdout-follow   = tail -n 100 -f @SIMULATION_NAME@.out @SIMULATION_NAME@.err