[orca]

# last-tested-on: 2013-11-04
# last-tested-by: Erik Schnetter <schnetter@gmail.com>

# Machine description
nickname        = orca
name            = Orca
location        = SHARCNET
description     = "SHARCNET's Orca cluster"
webpage         = https://www.sharcnet.ca/my/systems/show/73
status          = experimental

# Access to this machine
hostname        = orca.sharcnet.ca
rsynccmd        = /home/eschnett/rsync-3.0.9/bin/rsync
envsetup        = . /etc/profile.d/modules.sh && . /etc/profile.d/sharcnet-0-environment.sh && export MODULEPATH=/opt/sharcnet/modules && module load user-environment && module load mkl && module load openmpi/intel/1.6.2 && unset PETSC_DIR
aliaspattern    = ^orc-login\d+\.orca\.sharcnet$

# Source tree management
sourcebasedir   = /work/@USER@/orca
disabled-thorns = <<EOT
    ExternalLibraries/LORENE
        EinsteinInitialData/Meudon_Bin_BH
        EinsteinInitialData/Meudon_Bin_NS
        EinsteinInitialData/Meudon_Mag_NS
    ExternalLibraries/OpenSSL
        ExternalLibraries/curl
        ExternalLibraries/flickcurl
        LSUThorns/Flickr
        LSUThorns/Twitter
    ExternalLibraries/PETSc
        CactusElliptic/EllPETSc
        TAT/TATPETSc
EOT
#    McLachlan/ML_BSSN_O8
#    McLachlan/ML_BSSN_O8_Helper
#    McLachlan/ML_BSSN_MP_O8
#    McLachlan/ML_BSSN_MP_O8_Helper
optionlist      = orca-intel.cfg
submitscript    = orca.sub
runscript       = orca.run
# A user can have only 100 processes; more than -j8 leads to random
# failures from time to time
make            = make -j8

# Simulation management
basedir         = /scratch/@USER@/simulations
cpu             = AMD Opteron 6174 (Magny Cours) @ 2.2 GHz
cpufreq         = 2.2
flop/cycle      = 4
ppn             = 24
spn             = 4
# MPI appears to disallow threads spanning across multiple sockets.
max-num-threads = 6 # 24
# The queueing system enforces very long wait times when complete
# nodes are requested. When partial nodes are allowed, then cores are
# assigned randomly. Both make OpenMP infeasible.
num-threads     = 1 # 6
memory          = 32768
I1size          = 65536
I1linesize      = 64
I1assoc         = 2
I1cores         = 1
D1size          = 65536
D1linesize      = 64
D1assoc         = 2
D1cores         = 1
# D1stride      = 32768
L2size          = 524288
L2linesize      = 64
L2assoc         = 16
L2cores         = 1
# D2stride      = 32768
L3size          = 5240832
L3linesize      = 64
L3assoc         = 96
L3cores         = 6
# L3stride      = 54592
nodes           = 320
min-ppn         = 1 # 24
allocation      = NO_ALLOCATION
queue           = mpi   # DR_20349
maxwalltime     = 168:00:00
# For test jobs, use the "test" queue
#submit          = sqsub -q @QUEUE@ -f mpi -n @PROCS_REQUESTED@ --ppn=@(@PPN_USED@/@NUM_THREADS@)@ --tpp=@NUM_THREADS@ --mpp=@(@MEMORY@/(@PPN_USED@/@NUM_THREADS@))@M -W @WALLTIME_MINUTES@ @('@CHAINED_JOB_ID@' ne '' ? '-w @CHAINED_JOB_ID@' : '')@ -v -j @SIMULATION_NAME@ -m -o @RUNDIR@/@SIMULATION_NAME@.out -e @RUNDIR@/@SIMULATION_NAME@.err --nompirun @SCRIPTFILE@
submit          = qsub @SCRIPTFILE@
getstatus       = sqjobs @JOB_ID@
stop            = sqkill @JOB_ID@
#submitpattern   = submitted as jobid (\d+)
submitpattern   = (\d+)\.*
statuspattern   = '^@JOB_ID@[ \t]'
queuedpattern   = ' Q '
runningpattern  = ' R '
holdingpattern  = ' H '   # TODO