[hopper]

# last-tested-on: 2013-11-04
# last-tested-by: Erik Schnetter <schnetter@gmail.com>

# Machine description
nickname        = hopper
name            = Hopper
location        = NERSC
description     = The Cray XE6 at NERSC
webpage         = https://www.nersc.gov/systems/hopper-cray-xe6/
status          = production

# Access to this machine
hostname        = hopper.nersc.gov
rsynccmd        = /global/homes/s/schnette/rsync-3.0.9/bin/rsync
#envsetup        = module unload PrgEnv-cray PrgEnv-gnu PrgEnv-intel PrgEnv-pathscale PrgEnv-pgi && module load PrgEnv-cray && module load acml
#envsetup        = module unload PrgEnv-cray PrgEnv-gnu PrgEnv-intel PrgEnv-pathscale PrgEnv-pgi && module load PrgEnv-gnu && module load acml
envsetup        = <<EOT
    module unload PrgEnv-cray PrgEnv-gnu PrgEnv-intel PrgEnv-pathscale PrgEnv-pgi
    module load PrgEnv-intel
    module swap intel intel/13.1.3.192
    module load boost/1.54
    module load fftw/3.3.0.4
    module load gsl-intel/1.15
    module load cray-hdf5/1.8.11
    module load papi/5.1.2
EOT
#envsetup        = module unload PrgEnv-cray PrgEnv-gnu PrgEnv-intel PrgEnv-pathscale PrgEnv-pgi && module load PrgEnv-pathscale && module load acml
#envsetup        = module unload PrgEnv-cray PrgEnv-gnu PrgEnv-intel PrgEnv-pathscale PrgEnv-pgi && module load PrgEnv-pgi && module load acml
aliaspattern    = ^hopper\d+(\.nersc\.gov)?$

# Source tree management
sourcebasedir   = /project/projectdirs/m152/@USER@/hopper
disabled-thorns = <<EOT
    CaCUDA/CaCUDALib
    CaCUDA/WaveToyCUDA
    ExternalLibraries/curl
        ExternalLibraries/flickcurl
        LSUThorns/Twitter
    ExternalLibraries/git
    ExternalLibraries/HYPRE
    ExternalLibraries/libjpeg
        CactusIO/IOJpeg
    ExternalLibraries/libxml2
    ExternalLibraries/OpenSSL
        LSUThorns/Flickr
    ExternalLibraries/PETSc
        CactusElliptic/EllPETSc
        CarpetThorns/LSUPETSc
        CarpetThorns/LSUPoisson
        CarpetThorns/Lichnerowicz
        LSUDevelopment/WaveToyNoGhostsPETSc
        TAT/TATPETSc
EOT
#optionlist      = hopper-cray.cfg
#optionlist      = hopper-gcc.cfg
optionlist      = hopper-intel.cfg
#optionlist      = hopper-pathscale.cfg
#optionlist      = hopper-pgi.cfg
submitscript    = hopper.sub
runscript       = hopper.run
#make            = env LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/gcc/mpc/0.8.1/lib make -j8
make            = make -j8

# Simulation management
basedir         = /scratch/scratchdirs/@USER@/simulations
cpu             = 2 twelve-core AMD "MagnyCours" 2.1 GHz processors per node
cpufreq         = 2.1
flop/cycle      = 4
ppn             = 24
spn             = 4
max-num-threads = 24
num-threads     = 6             # strongly recommended by NERSC
memory          = 32768
I1size          = 65536
I1linesize      = 64
I1assoc         = 2
D1size          = 65536
D1linesize      = 64
D1assoc         = 2
L2size          = 524288
L2linesize      = 64
L2assoc         = 16
L3size          = 873728
L3linesize      = 64
L3assoc         = 96
nodes           = 6392
min-ppn         = 24
allocation      = NO_ALLOCATION
queue           = regular
maxwalltime     = 48:00:00   # large jobs >=683 nodes have 36:00:00
submit          = /opt/torque/4.2.7.h1/bin/qsub @SCRIPTFILE@
getstatus       = /opt/torque/4.2.7.h1/bin/qstat @JOB_ID@
stop            = /opt/torque/4.2.7.h1/bin/qdel @JOB_ID@
submitpattern   = (\d+)[.]
statuspattern   = ^@JOB_ID@[. ]
queuedpattern   = " Q "
runningpattern  = " R "
holdingpattern  = " H "
scratchbasedir   = /scratch/scratchdirs/@USER@
exechost        = /opt/torque/4.2.7.h1/bin/qstat -f @JOB_ID@
exechostpattern = exec_host = (\w+)/
stdout          = cat @JOB_ID@.hop.OU
stderr          = cat @JOB_ID@.hop.ER
stdout-follow   = tail -n 100 -f @JOB_ID@.hop.OU @JOB_ID@.hop.ER