[lincoln]

# Machine description
nickname        = lincoln
name            = Lincoln
location        = NCSA
description     = A Linux cluster with Tesla GPU accelerators at NCSA
webpage         = http://www.ncsa.illinois.edu/UserInfo/Resources/Hardware/Intel64TeslaCluster/
status          = production

# Access to this machine
hostname        = honest1.ncsa.uiuc.edu
rsynccmd        = /u/ac/eschnett/rsync-3.0.5/bin/rsync
envsetup        = export LM_LICENSE_FILE=1702@barbossa:1702@nani:1702@lilo:1702@stitch:1708@barbossa:1704@barbossa.ncsa.uiuc.edu:1055@abe-gw03.ncsa.uiuc.edu
aliaspattern    = ^(honest[1]|lincoln\d\d\d\d)(\.ncsa\.uiuc\.edu)?$

# Source tree management
sourcebasedir   = /u/ac/@USER@/lincoln
disabled-thorns = <<EOT
    ExternalLibraries/PETSc
        CactusElliptic/EllPETSc
        CarpetThorns/LSUPETSc
        CarpetThorns/LSUPoisson
        CarpetThorns/Lichnerowicz
        LSUDevelopment/WaveToyNoGhostsPETSc
        TAT/TATPETSc
EOT
optionlist      = lincoln-mvapich2.cfg
submitscript    = lincoln-mvapich2.sub
runscript       = lincoln-mvapich2.run
make            = make -j4

# Simulation management
basedir         = /u/ac/@USER@/scratch-global/lincoln/simulations
cpu             = Intel 64 (Harpertown)
cpufreq         = 2.33
flop/cycle      = 4
ppn             = 8
spn             = 2
mpn             = 1
max-num-threads = 8
num-threads     = 8
memory          = 16384
nodes           = 192
min-ppn         = 8
allocation      = out
# there is also a queue lincoln_debug with a walltime limit of 0:30:00
queue           = lincoln
maxwalltime     = 241:00:00
submit          = /usr/local/bin/qsub @SCRIPTFILE@
interactivecmd	= qsub -I -X -A @ALLOCATION@ -q @QUEUE@ -r n -l walltime=@WALLTIME@ -l nodes=@NODES@:ppn=@PPN@ -V -N @SHORT_SIMULATION_NAME@ -M @EMAIL@ -m abe
getstatus       = /usr/local/torque/bin/qstat @JOB_ID@
stop            = /usr/local/torque/bin/qdel @JOB_ID@
submitpattern   = (\d+)
statuspattern   = ^@JOB_ID@[. ]
queuedpattern   = " Q "
runningpattern  = " R "
holdingpattern  = " H "
scratchdir      = /cfs/scratch/users/${USER}/${PBS_JOBID}
exechost        = /usr/local/torque/bin/qstat -f @JOB_ID@
exechostpattern = exec_host = (\w+)/
stdout          = cat /u/ac/@USER@/.pbs_spool/@JOB_ID@.abem5.OU
stderr          = cat /u/ac/@USER@/.pbs_spool/@JOB_ID@.abem5.ER
stdout-follow   = tail -n 100 -f /u/ac/@USER@/.pbs_spool/@JOB_ID@.abem5.OU /u/ac/@USER@/.pbs_spool/@JOB_ID@.abem5.ER