[hopper]

# Machine description
nickname        = hopper
name            = Hopper
location        = NERSC
description     = A Cray XE6 at NERSC
webpage         = https://newweb.nersc.gov/for-users/computational-systems/hopper
status          = production

# Access to this machine
hostname        = hopper.nersc.gov
rsynccmd        = /global/homes/s/schnette/rsync-3.0.6/bin/rsync
#envsetup        = module unload PrgEnv-cray PrgEnv-gnu PrgEnv-intel PrgEnv-pathscale PrgEnv-pgi && module load PrgEnv-cray && module load acml
#envsetup        = module unload PrgEnv-cray PrgEnv-gnu PrgEnv-intel PrgEnv-pathscale PrgEnv-pgi && module load PrgEnv-gnu && module load acml
envsetup        = module unload PrgEnv-cray PrgEnv-gnu PrgEnv-intel PrgEnv-pathscale PrgEnv-pgi && module load PrgEnv-intel/4.0.36 && module swap intel intel/12.1.2.273 && module load acml/4.4.0 && module load ipm
#envsetup        = module unload PrgEnv-cray PrgEnv-gnu PrgEnv-intel PrgEnv-pathscale PrgEnv-pgi && module load PrgEnv-pathscale && module load acml
#envsetup        = module unload PrgEnv-cray PrgEnv-gnu PrgEnv-intel PrgEnv-pathscale PrgEnv-pgi && module load PrgEnv-pgi && module load acml
aliaspattern    = ^hopper\d+(\.nersc\.gov)?$

# Source tree management
sourcebasedir   = /project/projectdirs/m152/@USER@/hopper
disabled-thorns = <<EOT
    CaCUDA/CaCUDALib
    CaCUDA/WaveToyCUDA
    ExternalLibraries/curl
        ExternalLibraries/flickcurl
        LSUThorns/Twitter
    ExternalLibraries/git
    ExternalLibraries/HYPRE
    ExternalLibraries/libjpeg
        CactusIO/IOJpeg
    ExternalLibraries/libxml2
    ExternalLibraries/OpenSSL
        LSUThorns/Flickr
    ExternalLibraries/PETSc
        CactusElliptic/EllPETSc
        CarpetThorns/LSUPETSc
        CarpetThorns/LSUPoisson
        CarpetThorns/Lichnerowicz
        LSUDevelopment/WaveToyNoGhostsPETSc
        TAT/TATPETSc
EOT
#    Zelmani/ZelmaniLeak
#optionlist      = hopper-cray.cfg
#optionlist      = hopper-gcc.cfg
optionlist      = hopper-intel.cfg
#optionlist      = hopper-pathscale.cfg
#optionlist      = hopper-pgi.cfg
submitscript    = hopper.sub
runscript       = hopper.run
#make            = env LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/gcc/mpc/0.8.1/lib make -j8
make            = make -j8

# Simulation management
basedir         = /scratch/scratchdirs/@USER@/simulations
cpu             = 2 twelve-core AMD "MagnyCours" 2.1 GHz processors per node
cpufreq         = 2.1
flop/cycle      = 4
ppn             = 24
spn             = 4
max-num-threads = 24
num-threads     = 6             # strongly recommended by NERSC
memory          = 32768
I1size          = 65536
I1linesize      = 64
I1assoc         = 2
D1size          = 65536
D1linesize      = 64
D1assoc         = 2
L2size          = 524288
L2linesize      = 64
L2assoc         = 16
L3size          = 873728
L3linesize      = 64
L3assoc         = 96
nodes           = 6392
min-ppn         = 24
allocation      = m152
queue           = regular
maxwalltime     = 48:00:00   # large jobs >=683 nodes have 36:00:00
submit          = /usr/common/nsg/bin/qsub @SCRIPTFILE@
getstatus       = /usr/common/nsg/bin/qstat @JOB_ID@
stop            = /usr/common/nsg/bin/qdel @JOB_ID@
submitpattern   = (\d+)[.]sdb
statuspattern   = ^@JOB_ID@[. ]
queuedpattern   = " Q "
runningpattern  = " R "
holdingpattern  = " H "
scratchbasedir   = /scratch/scratchdirs/@USER@
exechost        = /usr/common/nsg/bin/qstat -f @JOB_ID@
exechostpattern = exec_host = (\w+)/
stdout          = cat @JOB_ID@.sdb.OU
stderr          = cat @JOB_ID@.sdb.ER
stdout-follow   = tail -n 100 -f @JOB_ID@.sdb.OU @JOB_ID@.sdb.ER