[kraken]

# Machine description
nickname        = kraken
name            = Kraken
location        = NICS
description     = A Cray XT5 at NICS
webpage         = http://www.nics.tennessee.edu/computing-resources/kraken
status          = production

# Access to this machine
hostname        = kraken-gsi2.nics.teragrid.org
rsynccmd        = /nics/b/home/eschnett/rsync-3.0.3/bin/rsync
sshcmd          = gsissh
sshopts         =
localsshsetup   = <<EOT
mkdir -p @SOURCEDIR@/.globus &&
: >> @SOURCEDIR@/.globus/proxy-teragrid &&
chmod go-rwx @SOURCEDIR@/.globus/proxy-teragrid &&
export X509_USER_PROXY=@SOURCEDIR@/.globus/proxy-teragrid &&
: mkdir -p @SOURCEDIR@/.globus/certificates-teragrid &&
: export X509_CERT_DIR=@SOURCEDIR@/.globus/certificates-teragrid &&
{
    {
        grid-proxy-info -issuer -file @SOURCEDIR@/.globus/proxy-teragrid 2>/dev/null | grep "^/C=US/O=National Center for Supercomputing Applications/" >/dev/null 2>&1 &&
        test $(grid-proxy-info -timeleft -file @SOURCEDIR@/.globus/proxy-teragrid 2>/dev/null) -gt 0 2>/dev/null
    } || {
        { grid-proxy-destroy 2>/dev/null || true; } &&
        myproxy-logon -p 7514 -s myproxy.teragrid.org -T -l @USER@ -o @SOURCEDIR@/.globus/proxy-teragrid
    }
} &&
{ globus-update-certificate-dir >/dev/null 2>&1 || true; }
EOT
envsetup        = source /etc/profile
aliaspattern    = ^kraken(-pwd)?[1234](\.nics\.utk\.edu)?$

# Source tree management
sourcebasedir   = /nics/a/proj/cactus/@USER@/xt5
optionlist      = kraken-intel.cfg
submitscript    = kraken.sub
runscript       = kraken.run
# These thorns need to be disabled for the PGI compiler; they work
# with the Intel compiler
#    EinsteinAnalysis/AHFinder
#    ExternalLibraries/OpenSSL
disabled-thorns = <<EOT
    ExternalLibraries/flickcurl
        LSUThorns/Flickr
    ExternalLibraries/git
    ExternalLibraries/HYPRE
EOT
enabled-thorns = <<EOT
    ExternalLibraries/PAPI
        Performance/PAPIClocks
EOT
make            =  env LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/opt/gcc/4.5.1/snos/lib64:/opt/gcc/mpc/0.8.1/lib" make -j2

# Simulation management
basedir         = /lustre/scratch/@USER@/simulations
cpu             = six-core AMD Opteron (Istanbul)
cpufreq         = 2.6
flop/cycle      = 4
ppn             = 12
spn             = 2
max-num-threads = 12
num-threads     = 6
memory          = 16384
I1size          = 65536
I1linesize      = 64
I1assoc         = 2
I1cores         = 1
D1size          = 65536
D1linesize      = 64
D1assoc         = 2
D1cores         = 1
L2size          = 524288
L2linesize      = 64
L2assoc         = 16
L2cores         = 1
L3size          = 5242880       # 5 MB; HT assist also uses 1 MB
L3linesize      = 64
L3assoc         = 48
L3cores         = 6
nodes           = 8256
min-ppn         = 12
allocation      = TG-MCA02N014
queue           = batch
maxwalltime     = 24:00:00
submit          = /opt/torque/2.4.8/bin/qsub @SCRIPTFILE@
getstatus       = /opt/torque/2.4.8/bin/qstat @JOB_ID@
stop            = /opt/torque/2.4.8/bin/qdel @JOB_ID@
submitpattern   = (\d+[.]nid[0-9]*)
statuspattern   = "^@JOB_ID@[. ].* [QRH] "
queuedpattern   = "^@JOB_ID@[. ].* Q "
runningpattern  = "^@JOB_ID@[. ].* R "
holdingpattern  = "^@JOB_ID@[. ].* H "
scratchdir      = /lustre/scratch/@USER@/scratch/@JOB_ID@
exechost        = /opt/torque/2.4.8/bin/qstat -f @JOB_ID@
exechostpattern = exec_host = (\w+)/
stdout          = cat @SIMULATION_NAME@.out
stderr          = cat @SIMULATION_NAME@.err
stdout-follow   = tail -n 100 -f @SIMULATION_NAME@.out @SIMULATION_NAME@.err