diff options
| -rw-r--r-- | README | 4 | ||||
| -rw-r--r-- | daily.sm (renamed from daily.sr) | 0 | ||||
| -rw-r--r-- | steuermann/hosts.ini | 61 | ||||
| -rw-r--r-- | steuermann/nodes.py | 7 | ||||
| -rw-r--r-- | steuermann/report.py | 3 | ||||
| -rw-r--r-- | steuermann/run.py | 80 | ||||
| -rw-r--r-- | steuermann/run_all.py | 23 | ||||
| -rw-r--r-- | steuermann/specfile.exy | 5 | ||||
| -rw-r--r-- | tests/hosts.py | 13 |
9 files changed, 116 insertions, 80 deletions
@@ -7,3 +7,7 @@ make python setup.py install +smc test.sm +? + + diff --git a/steuermann/hosts.ini b/steuermann/hosts.ini index c64e2aa..d51e9df 100644 --- a/steuermann/hosts.ini +++ b/steuermann/hosts.ini @@ -12,54 +12,67 @@ ; %(cmd)s - ; %(foo)s - if you include a foo= line ; + +; definitions common to various operating system environments + +[all] +local=[ 'sh', '-c', '%(script)s' ] + [linux:csh] hostname=no_such_machine run=[ 'ssh', '-q', '%(hostname)s', 'source .steuermann.%(hostname)s; cd %(workdir)s; hostname; %(script)s ' ] -workdir=/tmp +like=all + +[windows:cmd] +hostname=no_such_machine +run=[ 'python', '-m', 'steuermann.windows_comm', '%(hostname)s', '%(script)s' ] +like=all + +; machines defined named after each OS [rhe4-32] -hostname=herbert -like=linux:csh +like=herbert [rhe4-64] -hostname=thor -like=linux:csh +like=thor [rhe5-64] -hostname=arzach -like=linux:csh +like=arzach [leopard] -hostname=bond -like=linux:csh +like=bond [snow-leopard] -hostname=cadeau -like=linux:csh +like=cadeau -[arzach] -hostname=arzach +; actual machines + +[herbert] +hostname=herbert like=linux:csh +workdir=/herbert/data1/sienkiew/steuermann [thor] hostname=thor like=linux:csh +workdir=/thor/data2/sienkiew/steuermann +[arzach] +hostname=arzach +like=linux:csh +workdir=/arzach/data1/sienkiew/steuermann -[host_a] -run=[ 'sleep', '3' ] - -[host_b] -run=[ 'sleep', '3' ] - -[host_c] -run=[ 'sleep', '3' ] - -[host_d] -run=[ 'sleep', '3' ] +[bond] +hostname=bond +like=linux:csh +workdir=/Users/sienkiew/work/steuermann +[cadeau] +hostname=cadeau +like=linux:csh +workdir=/Users/sienkiew/work/steuermann ; There is a section [ALL] that is used with every machine name [ALL] diff --git a/steuermann/nodes.py b/steuermann/nodes.py index dfdb6d7..0173793 100644 --- a/steuermann/nodes.py +++ b/steuermann/nodes.py @@ -81,7 +81,7 @@ class command_tree(object): def add_command_list( self, table, hostlist, command_list ) : for host in hostlist : this_table = '%s:%s' % ( host, table ) - for command, script, after_clause, pos in command_list : + for command, script, script_type, after_clause, pos in command_list : # this happens once for each CMD clause # command is the name of this command # script is the script to run @@ -95,7 +95,7 @@ class command_tree(object): print "# warning: %s already used on line %s"%(command,self.node_index[command].input_line) # create the node - self.node_index[command]=node(command, script, nice_pos( current_file_name, pos) ) + self.node_index[command]=node(command, script, script_type, nice_pos( current_file_name, pos) ) for before_name, required, pos in after_clause : # this happens once for each AFTER clause @@ -178,12 +178,13 @@ def wildcard_name( wild, name ) : # true if the before node is finished running. # class node(object) : - def __init__(self, name, script, input_line) : + def __init__(self, name, script, script_type, input_line) : # the fully qualified name of the node self.name = name # the command script that this node runs self.script = script + self.script_type = script_type # what line of the input file specified this node; this is # a string of the form "foo.bar 123" diff --git a/steuermann/report.py b/steuermann/report.py index e1d2971..ca8d064 100644 --- a/steuermann/report.py +++ b/steuermann/report.py @@ -91,7 +91,8 @@ def get_table( db, run_name, tablename, info_callback, showdepth=0 ) : if cmd != prev_cmd : row = row + 1 t.set_value(row, 0, cmd) - t.set_value(row, 'depth', depth) + if showdepth : + t.set_value(row, 'depth', depth) prev_cmd = cmd info = info_callback( db, run_name, tablename, host, cmd ) diff --git a/steuermann/run.py b/steuermann/run.py index f3e86b3..1379120 100644 --- a/steuermann/run.py +++ b/steuermann/run.py @@ -23,24 +23,26 @@ class struct : class runner(object): # dict of all current running processes, indexed by node name - all_procs = { } + all_procs = None # index of nodes - node_index = { } - - # info about hosts we can run on - host_info = { } + node_index = None # dir where we write our logs logdir = '' + # + host_info_cache = None + ##### # def __init__( self, nodes, logdir ) : + self.all_procs = { } self.node_index = nodes self.load_host_info() self.logdir = logdir + self.host_info_cache = { } ##### # start a process @@ -53,16 +55,16 @@ class runner(object): print "....%s:%s/%s\n"%(node.host, node.table, node.cmd) try : - args = self.host_info[node.host] + args = self.get_host_info(node.host) except : print "ERROR: do not know how to run on %s"%node.host raise - run = args['run'] args = args.copy() args.update( script=node.script, + script_type=node.script_type, host=node.host, table=node.table, cmd=node.cmd @@ -73,7 +75,21 @@ class runner(object): for x in sorted([x for x in args]) : print '%s=%s'%(x,args[x]) - run = [ x % args for x in run ] + args['script'] = args['script'] % args + + if args['script_type'] == 'r' : + run = args['run'] + elif args['script_type'] == 'l' : + run = args['local'] + else : + raise Exception() + + t = [ ] + for x in run : + # bug: what to do in case of keyerror + t.append( x % args ) + + run = t if debug : print "RUN",run @@ -169,53 +185,37 @@ class runner(object): ##### - def _host_get_names( self, cfg, section, d ) : + def _host_get_names( self, cfg, section ) : + d = { } # pick all the variables out of this section for name, value in cfg.items(section) : - if name == 'run' : - # run is a list + if value.startswith('[') : + # it is a list d[name] = eval(value) else : # everything else is plain text d[name] = value + return d def load_host_info( self, filename=None ) : - self.host_info = { } # read the config file if filename is None : filename = os.path.dirname(__file__) + '/hosts.ini' - cfg = ConfigParser.RawConfigParser() - cfg.read(filename) - - # this dict holds the set of values that are defined as - # applying to all hosts - all_dict = { } - self._host_get_names(cfg, 'ALL', all_dict) + self.cfg = ConfigParser.RawConfigParser() + self.cfg.read(filename) - # for all the sections (except ALL) get the names from that section - for x in cfg.sections() : - if x == 'ALL' : - continue + def get_host_info(self, host) : + if not host in self.host_info_cache : - # start with a dict that contains what is in ALL - d = all_dict.copy() - - # get what there is to know about host x - self._host_get_names(cfg, x, d) - - # if it is like some other host, start over using ALL, then - # the LIKE host, then our own information + d = self._host_get_names(self.cfg, host) if 'like' in d : - like = d['like'] - d = all_dict.copy() - self._host_get_names(cfg, like, d) - self._host_get_names(cfg, x, d) + d1 = self.get_host_info(d['like']) del d['like'] + d1.update(d) + self.host_info_cache[host] = d1 + else : + self.host_info_cache[host] = d - print x,d - self.host_info[x] = d - - del cfg - + return self.host_info_cache[host] ##### diff --git a/steuermann/run_all.py b/steuermann/run_all.py index d94cafa..38f8241 100644 --- a/steuermann/run_all.py +++ b/steuermann/run_all.py @@ -37,15 +37,18 @@ def main() : # - di_nodes = nodes.read_file_list( sys.argv[2:] ) + all = sys.argv[1] == '-a' + if all : + di_nodes = nodes.read_file_list( sys.argv[2:] ) + else : + di_nodes = nodes.read_file_list( sys.argv[1:] ) xnodes = di_nodes.node_index run_name = str(datetime.datetime.now()).replace(' ','_') db = steuermann.config.open_db() register_database(db, run_name, xnodes) - n = sys.argv[1] - if n == '-a' : + if all : run_all(xnodes, run_name, db) else : run_interactive( xnodes, run_name, db ) @@ -55,6 +58,8 @@ def main() : def do_flag( xnodes, name, recursive, fn, verbose ) : if verbose : verbose = verbose + 1 + if not (':' in name ) and not ('/' in name) : + name = '*:*/'+name if not ':' in name : name = '*:' + name if ( '*' in name ) or ( '?' in name ) or ( '[' in name ) : @@ -180,7 +185,6 @@ def run_interactive( xnodes, run_name, db) : if keypress() : print "wait interrupted (processes continue)" break - print "wait finished" if keep_running : print "run step" @@ -279,7 +283,6 @@ def run_step( runner, xnodes, run_name, db ) : no_sleep = 0 # Loop, polling for work to do, or for finishing processes - print "loop" for x_name in xnodes : x=xnodes[x_name] @@ -317,7 +320,7 @@ def run_step( runner, xnodes, run_name, db ) : # of predecessors, we can run this one if released == len(x.released) : host, table, cmd = nodes.crack_name(x_name) - print "RUN", x_name + # print "RUN NODE", x_name db.execute("UPDATE status SET start_time = ?, status = 'S' WHERE ( run = ? AND host = ? AND tablename = ? AND cmd = ? )", ( str(datetime.datetime.now()), run_name, host, table, cmd ) ) @@ -330,7 +333,7 @@ def run_step( runner, xnodes, run_name, db ) : if not who_exited : break - print "SOMETHING EXITED",who_exited + # print "SOMETHING EXITED",who_exited # yes, something exited - no sleep, and keep running no_sleep = 1 keep_running = 1 @@ -344,7 +347,7 @@ def run_step( runner, xnodes, run_name, db ) : ( str(datetime.datetime.now()), who_exited[1], run_name, x_host, x_table, x_cmd ) ) db.commit() - runner.display_procs() + # runner.display_procs() return ( keep_running, no_sleep ) @@ -365,7 +368,7 @@ def keypress() : ##### -def info_callback_want( tablename, cmd, host, status ) : +def info_callback_want( db, run, tablename, host, cmd ) : n = xnodes['%s:%s/%s'%(host,tablename,cmd)] s = '' if n.skip : @@ -376,7 +379,7 @@ def info_callback_want( tablename, cmd, host, status ) : s = '-' return s -def info_callback_depth( tablename, cmd, host, status ) : +def info_callback_depth( db, run, tablename, host, cmd ) : n = xnodes['%s:%s/%s'%(host,tablename,cmd)] return n.depth diff --git a/steuermann/specfile.exy b/steuermann/specfile.exy index 8dd42a3..5dc715f 100644 --- a/steuermann/specfile.exy +++ b/steuermann/specfile.exy @@ -20,6 +20,7 @@ parser specfile: token tablename: "[a-zA-Z0-9_.-]+" token wildname: "[*?a-zA-Z0-9_.-]+" token RUN: "RUN" + token LOCAL: "LOCAL" token string: '"[^"]*"' token SLASH: "/" token COLON: ":" @@ -49,9 +50,9 @@ parser specfile: rule command: # a single command, including any number of AFTER clauses CMD {{ cmd_pos = self._scanner.get_pos() }} cmdname {{ cmd_name=cmdname; script=cmdname; x_after_clause = [ ] }} - [ RUN string {{ script = string[1:-1] }} ] + [ RUN string {{ script = string[1:-1]; script_type='r' }} | LOCAL string {{ script = string[1:-1]; script_type='l' }} ] ( {{ after_pos = self._scanner.get_pos() }} AFTER optword after_spec {{ x_after_clause.append( (after_spec,optword, after_pos) ) }} )* - {{ return ( cmd_name, script, x_after_clause, cmd_pos ) }} + {{ return ( cmd_name, script, script_type, x_after_clause, cmd_pos ) }} # in the AFTER clause, you can say OPT to mean the node is optional (not an error if it does not exist) rule optword: diff --git a/tests/hosts.py b/tests/hosts.py new file mode 100644 index 0000000..5fbfa25 --- /dev/null +++ b/tests/hosts.py @@ -0,0 +1,13 @@ +import steuermann.run as run + +r = run.runner(0,0) + +def test_1() : + print r.cfg.sections() + +def test_2() : + print r.get_host_info('rhe4-32') + + +test_1() +test_2() |