# simlib.py -- shared necessary library functions across all sim-* utilties # # # # import os, sys, re import libutil import socket import traceback import os.path import simsubs from libutil import dprint class SimLib: def __init__(self, env): # we need SimEnvironment self.SimEnvironment = env self.OptionsManager = self.SimEnvironment.OptionsManager self.ConfigurationDatabase = self.SimEnvironment.ConfigurationDatabase self.DefineDatabase = self.SimEnvironment.DefineDatabase def GetHostNameAlias(self): # first check if our hostname was specified via an option if self.OptionsManager.HasOption('hostname'): return self.OptionsManager.GetOption('hostname') # get the users home directory, check for ~/.hostname homedir = os.environ['HOME'] home_hostname = "%s%s%s" % (homedir, os.sep, ".hostname") if libutil.FileExists(home_hostname): hostname = self.GetFileContents(home_hostname).strip() return hostname # lets get the hostname hostname = os.popen("hostname").read().strip() try: (name, aliases, _) = socket.gethostbyname_ex(hostname) except: if self.SimEnvironment.VERBOSE: dprint("Warning: could not resolve hostname %s" % hostname) return hostname # out of all our names, find one that has a dot, if possible. names = aliases names.insert(0, name) names.insert(0, hostname) names = filter(lambda n: not (re.match("localhost", n)), names) for n in names: if n.count("."): return n if len(name) > 0: return name return hostname def BoolToYesNo(self, val): if val == True: return 'yes' return 'no' def GetLocalEnvironment(self): machine = self.SimEnvironment.LocalMachine machineEntry = self.SimEnvironment.LocalMachineEntry req_keys = ['user', 'sourcebasedir'] self.VerifyKeys(machineEntry, req_keys) sourceBaseDir = self.GetLocalSourceBaseDir() path = self.GetDirSuffix(sourceBaseDir) return (machine, machineEntry, sourceBaseDir, path) def GetMachineName(self): if self.OptionsManager.HasOption("machine"): return self.OptionsManager.GetOption("machine") machineName = self.GetHostNameAlias() machines = self.ConfigurationDatabase.GetMachines() found_machines = [] for m in machines: if m == machineName: found_machines.append(m) else: entry = self.ConfigurationDatabase.GetMachine(m) if entry.HasKey('hostname') and entry.hostname == machineName: found_machines.append(m) else: if entry.HasKey("aliaspattern"): if libutil.Matches(entry.aliaspattern, machineName): found_machines.append(m) error = None if len(found_machines) == 0: error = "Unknown machine name %s" % machineName if len(found_machines) > 1: error = "Could not identify machine %s -- possible matches are %s" % (machineName, found_machines) if error: if self.SimEnvironment.VERBOSE: dprint(error) return None return found_machines[0] def MachineExists(self, machineName): machines = self.ConfigurationDatabase.GetMachines() found_machines = [] for m in machines: if m == machineName: found_machines.append(m) else: entry = self.ConfigurationDatabase.GetMachine(m) if entry.HasKey('hostname') and entry.hostname == machineName: found_machines.append(m) else: if entry.HasKey("aliaspattern"): if libutil.Matches(entry.aliaspattern, machineName): found_machines.append(m) if len(found_machines) == 0: return False if len(found_machines) > 1: return False return True # Get source base directory def GetLocalSourceBaseDir(self): if self.OptionsManager.HasOption('sourcebasedir'): sourcebasedir = self.OptionsManager.GetOption('sourcebasedir') else: return self.GetSourceBaseDir(self.SimEnvironment.LocalMachineEntry) if not(sourcebasedir[0] == "/"): cwd = os.getcwd() sourcebasedir = "%s%s%s" % (cwd, os.sep, sourcebasedir) if not(os.path.exists(sourcebasedir)): dprint('Cannot access source base directory "%s"' % sourcebasedir) return sourcebasedir def BaseName(self, filename): (path, file) = os.path.split(filename) return file def FileBaseName(self, filename): # simulate rsplit, because rsplit doesn't exist # in python 2.3 parts = self.BaseName(filename).split(".") parts.pop() return ".".join(parts) return file def ParseSimulationCommandLine(self): parfile = None simulationName = None walltime = None procs = None #first, lets count how many arguments we have. numArgs = len(self.OptionsManager.args) if numArgs < 3: if numArgs == 1: argument = self.OptionsManager.args.pop(0) if argument.count(".par") > 0: parfile = argument dprint("Parfile: %s" % parfile) self.OptionsManager.UpdateOption('parfile', parfile) return self.FileBaseName(parfile) return argument else: return None argument = self.OptionsManager.args.pop(0) # 4 args -- simname, parfile, procs, walltime if numArgs == 4: simulationName = argument argument = self.OptionsManager.args.pop(0) if argument.count(".par") > 0: parfile = argument dprint("Parfile: %s" % parfile) self.OptionsManager.UpdateOption('parfile', parfile) if simulationName == None: simulationName = self.FileBaseName(parfile) else: dprint("Error: argument stream in incorrect format.") dprint("Expecting: %s %s [simulationName] parfile.par procs walltime" % (sys.argv[0], self.SimEnvironment.COMMAND)) sys.exit(1) dprint("Simulation Name: %s" % simulationName) argument = self.OptionsManager.args.pop(0) self.OptionsManager.UpdateOption('procs', int(argument)) dprint("Procs: %s" % argument) argument = self.OptionsManager.args.pop(0) self.OptionsManager.UpdateOption('walltime', argument) dprint("Walltime: %s" % argument) return simulationName def GetBaseDir(self, machineEntry): if self.OptionsManager.HasOption('basedir'): basedir = self.OptionsManager.GetOption('basedir') else: machineEntry = self.SimEnvironment.LocalMachineEntry self.VerifyKeys(machineEntry, ['user', 'basedir']) user = machineEntry.user self.DefineDatabase.Set('USER', user) basedir = self.DefineDatabase.SubAll(machineEntry.basedir) if not(basedir[0] == "/"): cwd = os.getcwd() basedir = "%s%s%s" % (cwd, os.sep, basedir) #if not(os.path.exists(basedir)): # dprint('Cannot access source base directory "%s"' % basedir) return basedir def GetSourceBaseDir(self, machineEntry): if not(machineEntry.HasKey('user')): machineName = machineEntry.nickname dprint("Error: machine entry %s does not have a user defined" % machineName) sys.exit(1) user = machineEntry.user self.DefineDatabase.Set('USER', user) sourcebasedir = self.DefineDatabase.SubAll(machineEntry.sourcebasedir) if not(sourcebasedir[0] == "/"): cwd = os.getcwd() sourcebasedir = "%s%s%s" % (cwd, os.sep, sourcebasedir) #if not(os.path.exists(sourcebasedir)): # dprint('Cannot access source base directory "%s"' % sourcebasedir) return sourcebasedir def GetDirSuffix(self, prefix): if not(os.path.exists(prefix)): dprint("Configured sourcebasedir \"%s\" does not exist or is not readable" % prefix) sys.exit(1) dir = os.getcwd() os.chdir(prefix) real_prefix = os.getcwd() os.chdir(dir) pattern = "^%s/(.*)$" % real_prefix p = re.compile(pattern) matches = p.match(dir) if matches == None: error = "Called from the wrong location." error = "%s\n%s" % (error, "Current directory is '%s'" % dir) error = "%s\n%s" % (error, "but expected a subdirectory of '%s'." % prefix) error = "%s\n%s" % (error, "It is also be possible that you need to correct your 'sourcebasedir' entry") error = "%s\n%s" % (error, "in the mdb entry for this machine.") dprint(error) sys.exit(1) return matches.group(1) def GetAppPath(self): app_marker = self.ConfigurationDatabase.GetConfigOption("app-marker") # look for APP_PATH if os.environ.has_key('SIMFACTORY_APP_PATH'): env_path = os.environ['SIMFACTORY_APP_PATH'] if not(os.path.exists(self.BuildPath([env_path, app_marker]))): dprint("Error: SIMFACTORY_APP_PATH %s is not a valid App source location" % env_path) else: #dprint("Cactus path env: %s" % env_path) return env_path # lets take a look at os.cwd() current_path = os.getcwd() #dprint("DEBUG: current_path: %s" % current_path) if os.path.exists(self.BuildPath([current_path, app_marker])): #dprint("DEBUG: Cactus path 2: %s" % current_path) return current_path # alright, still no luck. Lets see if we're in a subdirectory of the Cactus source tree parts = current_path.split(os.sep) current_path = "" for part in parts: if len(current_path) == 0: current_path = "%s%s" % (os.sep, part) else: current_path = "%s%s%s" % (current_path, os.sep, part) #dprint("DEBUG: current_path: %s" % current_path) if os.path.exists(self.BuildPath([current_path, app_marker])): #dprint("DEBUG: Cactus path 3: %s" % current_path) return current_path # try using BASE_PATH to determine whether or not simfactory is # installed in the Cactus source tree. basepath = self.SimEnvironment.BASE_PATH #dprint("DEBUG: basepath: %s" % basepath) parts = basepath.split(os.sep) parts.pop() path = os.sep.join(parts) #dprint("DEBUG: after pop, path is: %s" % path) if os.path.exists(self.BuildPath([path, app_marker])): #dprint("DEBUG: Cactus path 1: %s" % path) return path return None def GetUsername(self): if "USER" not in os.environ: username = os.popen("whoami").read() else: username = os.environ["USER"] return username def GetSimulations(self): pass # need basedir, simulationdir, and internaldir machine = self.SimEnvironment.LocalMachine machineEntry = self.SimEnvironment.LocalMachineEntry base_dir = self.GetBaseDir(machineEntry) #simulation_dir = self.BuildPath([base_dir, self.SimulationName]) sims = [] if not(os.path.exists(base_dir)): return sims for simulation in os.listdir(base_dir): simpath = self.BuildPath([base_dir, simulation]) if os.path.isdir(simpath): if simulation not in ['CACHE', 'TRASH']: sims.append(simulation) #dprint("Simulations: %s" % sims) return sims def GetConfigurations(self): config_dir = self.BuildPath([self.SimEnvironment.APP_PATH, "configs"]) ll = [] if not(os.path.exists(config_dir)): return ll for item in os.listdir(config_dir): itempath = self.BuildPath([config_dir, item]) if os.path.isdir(itempath): ll.append(item) #dprint("Configurations: %s" % ll) return ll def GetSSHCommand(self, machine, cmd, options=None): if options == None: options = '' currentMachine = machine while currentMachine != None: entry = self.ConfigurationDatabase.GetMachine(currentMachine) req_key = ['sshcmd', 'sshopts', 'user', 'hostname'] self.VerifyKeys(entry, req_key) sshcmd = entry.sshcmd sshopts = entry.sshopts user = entry.user hostname = entry.hostname self.DefineDatabase.Set('USER', user) ssh = "%s %s %s %s@%s" % (sshcmd, sshopts, options, user, hostname) ssh = self.DefineDatabase.SubAll(ssh) if entry.HasKey("sshsetup"): cmd = "{ %s; } && %s" % (entry.sshsetup, cmd) cmd = self.QuoteSafe(cmd) cmd = "%s \\\"/bin/bash -c %s\\\"" % (ssh, cmd) trampoline = self.GetTrampoline(currentMachine) if trampoline != None: if not(self.ConfigurationDatabase.HasMachineEntry(trampoline)): dprint("Unknown trampoline %s specified for machine %s" % (trampoline, currentMachine)) sys.exit(1) tentry = self.ConfigurationDatabase.GetMachine(trampoline) if tentry.HasKey("localsshsetup"): cmd = "{ %s; } && %s" % (self.DefineDatabase.SubAll(tentry.localsshsetup), cmd) currentMachine = trampoline return cmd def ExecuteCommand(self, command, output=False): sys.stdout.flush() sys.stderr.flush() if command == None or len(command) == 0: return environcmd = self.GetMachineOption('environcmd') command = "/bin/bash -c \"{ %s; } && %s\"" % (environcmd, command) if output == False: dprint("Executing: %s\n" % command) if output == False: ret = os.system(command) if ret != 0: dprint("Error %s occured while executing command \"%s\"" % (ret, command)) return ret else: command = "%s 2>&1" % command output = os.popen(command).read() return output def ExecuteReplaceCommand(self, command, output=False): if output == False: dprint("Executing: %s\n" % command) environcmd = self.GetMachineOption('environcmd') command = "{ %s; } && %s" % (environcmd, command) sys.stdout.flush() sys.stderr.flush() pid = os.fork() if pid > 0: retinfo = os.wait() return retinfo[1] else: args = ['bash', '-c', command] os.execv('/bin/bash', args) def GetArguments(self): args = [] if self.OptionsManager.HasOption("argument"): args.append(self.OptionsManager.GetOption('argument')) return args def QuoteSafe(self, cmd): return "'%s'" % cmd.replace("'", r"'\''") def VerifyKeys(self, machineEntry, req_keys): for key in req_keys: if not(machineEntry.HasKey(key)): dprint("Error: machine %s is missing a required key: %s" % (machineEntry.EntryName, key)) sys.exit(1) def GetIOMachine(self, machineName): return self.GetMachineByKey(machineName, 'iomachine') def GetTrampoline(self, machineName): return self.GetMachineByKey(machineName, 'trampoline', True) def GetMachineByKey(self, machineName, key, useNone=False): entry = self.ConfigurationDatabase.GetMachine(machineName) if not(entry.HasKey(key)): if useNone: return None else: return machineName mm = entry.GetKey(key) if not(self.ConfigurationDatabase.HasMachineEntry(mm)): dprint("Error: specified %s %s for machine %s does not exist" % (key, mm, machineName)) sys.exit(1) return mm def GetUseNodes(self): if not(self.OptionsManager.HasOption('use-nodes')): return [] nodes = self.OptionsManager.GetOption('use-nodes') return nodes.split(",") def GetDefaultConfiguration(self): OptionsManager = self.OptionsManager ConfigurationDatabase = self.ConfigurationDatabase dEntry = ConfigurationDatabase.GetConfigOption("default-configuration-name") if dEntry == None: configName = "sim" else: configName = dEntry config = configName opts = ['debug', 'optimise', 'unsafe', 'profile'] for opt in opts: nopt = "no%s" % opt if self.OptionsManager.RawOptionDefined(opt): config = "%s-%s" % (config, opt) if self.OptionsManager.RawOptionDefined(nopt): config = "%s-%s" % (config, nopt) return config def FileExists(self, filename): return os.path.exists(filename) # TOOD: How to make this abort if there is an error? def GetFileContents(self, filename, default="", suppressWarnings=True): if filename == None: return default try: fptr = open(filename, "r") contents = fptr.read() fptr.close() return contents except: if suppressWarnings == False: dprint("Warning: could not open %s for reading" % filename) return default def WriteContents(self, filename, contents): try: fptr = open(filename, "w") except: dprint("Error: could not open %s for writing" % filename) sys.exit(1) fptr.write(contents) fptr.close() def RemoveFile(self, filename): try: os.unlink(filename) except OSError: return def RenameFile(self, old, new): try: os.rename(old, new) except OSError: return def GetVersion(self, fileContents): regex = 'VERSION\s*=\s*([^#;\n]*)[#;]?.*' rr = re.compile(regex) m = rr.search(fileContents) if m != None: return m.group(1) else: return "" def BuildPath(self, args): return os.sep.join(args) def GetOptionsList(self, required=False): return self.GetEtcFile("optionlist", required) def GetThornList(self, required=False): return self.GetEtcFile("thornlist", required) def GetThornListContents(self, filename): machineEntry = self.SimEnvironment.LocalMachineEntry disabledThorns = machineEntry.GetKey('disabled-thorns') defineDatabase = simsubs.DefineDatabase(self.SimEnvironment) if disabledThorns != None: for thorn in disabledThorns.split(): defineDatabase.AddSubstitution(r'^\s*(%s)\b' % thorn, '# @1@') contents = self.GetFileContents(filename, None) if contents == None: dprint("Error: could not open file '%s' for reading" % filename) sys.exit(1) contents = defineDatabase.SubAll(contents) return contents def GetParFile(self, required=False): return self.GetEtcFile("parfile", required) def GetSubmitScript(self, required=False): return self.GetEtcFile("submitscript", required) def GetRunScript(self, required=False): return self.GetEtcFile("runscript", required) def GetMachineOption(self, option, defaultValue=None): value = None if self.OptionsManager.HasOption(option): value = self.OptionsManager.GetOption(option) return value else: value = defaultValue if value == None: # make sure option is set self.VerifyKeys(self.SimEnvironment.LocalMachineEntry, [option]) value = self.SimEnvironment.LocalMachineEntry.GetKey(option) if value == defaultValue: if self.SimEnvironment.VERBOSE: dprint("DEBUG: for machine option \"%s\", using default value \"%s\"" % (option, value)) return value def GetProcs(self, existingProperties, procs_arg=None): machineEntry = self.SimEnvironment.LocalMachineEntry minppn = int(machineEntry.GetKey("min-ppn")) maxppn = int(machineEntry.GetKey("ppn")) maxnodes = int(machineEntry.GetKey("nodes")) procs = None if existingProperties != None: procs = int(self.GetMachineOption('procs', existingProperties.procs)) else: if procs_arg != None: procs = procs_arg else: procs = int(self.GetMachineOption('procs', 1)) try: procs = int(procs) except ValueError: dprint("couldn't coerse procs value %s to an integer" % procs) sys.exit(1) if procs < 1: dprint("Illegal number of processors specified for the simulation") sys.exit(1) # writer procs writer_procs = 1 if existingProperties != None: if existingProperties.HasProperty('writerprocs'): writer_procs = existingProperties.writerprocs if self.OptionsManager.HasOption('writer-procs'): writer_procs = self.OptionsManager.GetOption('writer-procs') try: writer_procs = int(writer_procs) except ValueError: dprint("couldn't coerse writer-procs value %s to an integer" % writer_procs) sys.exit(1) if writer_procs < 1: dprint("Illegal number of writer processors specified for the adcirc simulation") sys.exit(1) #ppn if existingProperties != None: ppn = int(self.GetMachineOption('ppn', existingProperties.ppn)) else: ppn = int(self.GetMachineOption('ppn')) if ppn == None: ppn = maxppn if ppn > procs: ppn = procs #if (defined $minppn && ($ppn < $minppn || $ppn > $maxppn)) { #die "Illegal number of requested processors per node specified: specified ppn=$ppn (min-ppn is $minppn, max-ppn is $maxppn)"; if minppn != None and (ppn < minppn or ppn > maxppn): dprint("Illegal number of requested processors per node specified: specified ppn=%s (min-ppn is %s, max-ppn is %s)" % (ppn, minppn, maxppn)) sys.exit(1) ppnused = None if self.OptionsManager.HasOption('ppn-used'): ppnused = self.OptionsManager.GetOption('ppn-used') if ppnused == None: if existingProperties != None: ppnused = existingProperties.ppnused else: ppnused = ppn try: ppnused = int(ppnused) except ValueError: dprint("couldn't coerse ppn-used value %s to an integer" % ppnused, libutil.ALWAYS_PRINT) sys.exit(1) if ppnused < 1: dprint("Illegal number of used processors per node specified: specified ppn-used=%s" % ppnused, libutil.ALWAYS_PRINT) sys.exit(1) if ppnused > ppn: dprint("WARNING: Too many used processors per node specified: specified ppn-used=%s (ppn is %s)" % (ppnused, ppn)) if existingProperties != None: num_threads = self.GetMachineOption('num-threads', existingProperties.numthreads) else: num_threads = self.GetMachineOption('num-threads', 1) try: num_threads = int(num_threads) except ValueError: dprint("couldn't coerse num_threads value %s to an integer" % num_threads, libutil.ALWAYS_PRINT) sys.exit(1) if num_threads < 1: dprint("Illegal number of threads per process specified: specified num-threads=%s" % num_threads, libutil.ALWAYS_PRINT) sys.exit(1) if num_threads > procs: dprint("WARNING: Too many threads per process specified: specified num-threads=%s (ppn-used is %s)" % (num_threads, ppnused)) num_procs = procs / int(num_threads) if num_procs < 1: num_procs = 1 nodes = int((procs + ppnused - 1) / ppnused) if nodes < 1: nodes = 1 if nodes > maxnodes: dprint("Too many nodes specified: nodes=%s (maxnodes is %s)" % (nodes, maxnodes)) procs_requested = nodes * ppn if num_procs < writer_procs: dprint("Error: %s processors is not enough to use %s processors for writing" % (num_procs, writer_procs), libutil.ALWAYS_PRINT) sys.exit(1) if (num_procs - writer_procs) <= 0: dprint("Error: not enough available processors, %s total processors specified with %s allocated to writing" % (num_procs, writer_procs), libutil.ALWAYS_PRINT) sys.exit(1) return (nodes, ppnused, procs, ppn, procs_requested, num_procs, num_threads, writer_procs) def GetEtcFile(self, option, required=False): if self.OptionsManager.HasOption(option): ff = self.OptionsManager.GetOption(option) if len(ff) == 0 or ff == None: if required == True: dprint("Error: specified %s is empty or None" % option) sys.exit(1) else: return None if not(os.path.exists(ff)): if not(ff.startswith("/")): folder = "%ss" % option ff = self.BuildPath((self.SimEnvironment.ETC_PATH,folder,ff)) if not(os.path.exists(ff)): dprint("Error: specified %s %s does not exist or is not readable" % (option, ff)) sys.exit(1) return os.path.abspath(ff) machineEntry = self.SimEnvironment.LocalMachineEntry if required == True: # make sure option is set self.VerifyKeys(machineEntry, [option]) ff = machineEntry.GetKey(option) if ff == None: if required == True: dprint("Error: specified %s %s does not exist or is not readable" % (option, ff)) sys.exit(1) else: return None if ff[0] != os.sep: folder = "%ss" % option eff = self.BuildPath((self.SimEnvironment.ETC_PATH,folder,ff)) dprint("looking for etcfile: %s" % eff) if os.path.exists(eff): ff = eff if not(os.path.exists(ff)): if required == True: dprint("Error: specified %s %s does not exist or is not readable" % (option, ff)) sys.exit(1) else: ff = None return ff