diff options
| -rw-r--r-- | daily.in | 76 | ||||
| -rw-r--r-- | scripts/steuermann_report.cgi | 36 | ||||
| -rw-r--r-- | steuermann/nodes.py | 32 | ||||
| -rw-r--r-- | steuermann/report.py | 208 | ||||
| -rw-r--r-- | steuermann/run.py | 2 | ||||
| -rw-r--r-- | steuermann/run_all.py | 142 | ||||
| -rw-r--r-- | tests/y.in | 4 | ||||
| -rw-r--r-- | x.in | 9 |
8 files changed, 307 insertions, 202 deletions
diff --git a/daily.in b/daily.in new file mode 100644 index 0000000..ae0c843 --- /dev/null +++ b/daily.in @@ -0,0 +1,76 @@ +# one approach we might use for ssb builds + +TABLE assemble HOST thor + CMD svnsync RUN "svnsync" + AFTER OPT assemble/irafx_update + CMD stsci_python RUN "assemble_stsci_python" + AFTER svnsync + CMD stsci_iraf RUN "assemble_stsci_iraf" + AFTER svnsync + CMD astrolib RUN "assemble_astrolib" + AFTER svnsync + CMD axe RUN "assemble_axe" + AFTER svnsync + CMD hstcal RUN "assemble_hstcal" + AFTER svnsync + +# all machines +TABLE build HOST rhe4-32 rhe4-64 rhe5-64 leopard snow_leopard + + CMD python2.7 RUN "build_stsci_python dev2.7" + AFTER *:assemble/stsci_python + + CMD hstcal RUN "build_hstcal" + AFTER *:assemble/hstcal + +TABLE build HOST rhe-64 + + CMD python2.6 RUN "build_stsci_python dev2.6" + AFTER *:assemble/stsci_python + + CMD python2.5 RUN "build_stsci_python dev2.5" + AFTER *:assemble/stsci_python + + +# 32 bit +TABLE build32 HOST rhe4-32 leopard + + CMD stsci_iraf RUN "build_stsci_iraf" + AFTER *:assemble/stsci_iraf + + CMD axe RUN "build_axe" + AFTER *:assemble/axe + AFTER build32/stsci_iraf + +# copy 32 bit exe to 64 bit machine +TABLE build64 HOST rhe4-64 + CMD iraf32_hack RUN "iraf32hack rhe4-32" + +TABLE build64 HOST rhe5-64 + CMD iraf32_hack RUN "iraf32hack rhe4-32" + +TABLE build64 HOST snow_leopard + CMD iraf32_hack RUN "iraf32hack leopard" + + +# +TABLE build_finished HOST rhe4-32 leopard + CMD finished2.7 RUN "echo done" + AFTER build/python2.7 + AFTER build/hstcal + AFTER build32/stsci_iraf + AFTER build32/axe + +TABLE build_finished HOST rhe4-64 rhe5-64 snow_leopard + CMD finished2.7 RUN "echo done" + AFTER build/python2.7 + AFTER build/hstcal + AFTER build64/iraf32_hack + + +# +TABLE web_updates HOST thor + CMD pyraf RUN "exit 1" + AFTER *:assemble/svnsync + CMD pyfits RUN "exit 1" + AFTER *:assemble/svnsync diff --git a/scripts/steuermann_report.cgi b/scripts/steuermann_report.cgi new file mode 100644 index 0000000..fb82b89 --- /dev/null +++ b/scripts/steuermann_report.cgi @@ -0,0 +1,36 @@ +#! python + +import cgi +import cgitb + +cgitb.enable() + +form = cgi.FieldStorage(keep_blank_values=1) +cginame = os.getenv("SCRIPT_NAME") + +import sqlite3 + +if not 'action' in form : + print 'content-type: text/html' + print '' + db = sqlite3.connect('db.sr') + c = db.cursor() + c.execute('SELECT DISTINCT run FROM status ORDER BY run ASC') + for run, in c : + print "<a href=%s?action=report&run=%s>%s</a><br>"%(cginame, run, run) + return + +action = form['action'].value + +if action == 'report' : + import steuermann.report + print 'content-type: text/html' + print '' + run = form['run'].value + print steuermann.report.report_html( db, run ) + return + +print 'content-type: text/html' +print '' +print 'no action?' + diff --git a/steuermann/nodes.py b/steuermann/nodes.py index 5e25d8f..dfdb6d7 100644 --- a/steuermann/nodes.py +++ b/steuermann/nodes.py @@ -40,16 +40,21 @@ class command_tree(object): if ( fnmatch.fnmatchcase(hl,host ) and fnmatch.fnmatchcase(tl,table) and fnmatch.fnmatchcase(cl,cmd ) ) : - self.connect(x, after, required, pos) + # yes, the wild card matches this one; connect them + # (but don't let a node come before itself because the wild card is too loose) + if x != after : + self.connect(x, after, required, pos) else : self.connect(before, after, required, pos) - # work out the depths of each node + # Work out the depths of each node. + # Since we are walking the graph, we also detect dependency loops here. compute_depths( self.node_index ) # make the actual connection between nodes - def connect( self, after, before, required, line ) : + def connect( self, before, after, required, line ) : + if not after in self.node_index : if required : print "error: %s happens after non-existant %s - line %s"%(before,after,line) @@ -76,7 +81,7 @@ class command_tree(object): def add_command_list( self, table, hostlist, command_list ) : for host in hostlist : this_table = '%s:%s' % ( host, table ) - for command, script, after, pos in command_list : + for command, script, after_clause, pos in command_list : # this happens once for each CMD clause # command is the name of this command # script is the script to run @@ -92,7 +97,7 @@ class command_tree(object): # create the node self.node_index[command]=node(command, script, nice_pos( current_file_name, pos) ) - for before_name, required, pos in after : + for before_name, required, pos in after_clause : # this happens once for each AFTER clause # before is the name of a predecessor that this one comes after # required is a boolean, whether the predecessor must exist @@ -237,6 +242,10 @@ def nice_pos( filename, yapps_pos ) : def c_d_fn(x,depth) : + if x.in_recursion : + print "error: loop detected at",x.name + return + # if it is already deeper than where we are now, we can (must) # prune the tree walk here. if x.depth >= depth : @@ -244,9 +253,11 @@ def c_d_fn(x,depth) : if depth > 100 : # bug: proxy for somebody wrote a loop - print "error: depth > 100" + print "error: depth > 100, node = ",x.name return + x.in_recursion = 1 + # assign the depth x.depth = depth @@ -255,6 +266,8 @@ def c_d_fn(x,depth) : for y in x.successors : c_d_fn(y,depth) + x.in_recursion = 0 + def compute_depths(nodes) : # init everything @@ -262,6 +275,7 @@ def compute_depths(nodes) : x = nodes[x] x.depth = 0 x.successors = [ ] + x.in_recursion = 0 # walk the nodes in an arbitrary order; make a list of successors for x in nodes : @@ -275,6 +289,11 @@ def compute_depths(nodes) : for x in nodes : c_d_fn(nodes[x],1) + # + for x in nodes : + x = nodes[x] + del x.in_recursion + ##### @@ -287,7 +306,6 @@ def read_file_list( file_list ) : di = command_tree( ) for x in file_list : current_file_name = x - print x sc = specfile.specfileScanner( open(x,'r').read() ) p = specfile.specfile( scanner=sc, data=di ) result = exyapps.runtime.wrap_error_reporter( p, 'start' ) diff --git a/steuermann/report.py b/steuermann/report.py index 1bf4f6a..e1df5ce 100644 --- a/steuermann/report.py +++ b/steuermann/report.py @@ -1,146 +1,92 @@ ''' -prototype of report - +Generate reports from the database + ''' -import subprocess + import time import sys +import sqlite3 +import pandokia.text_table as text_table +import StringIO -try : - import CStringIO as StringIO -except ImportError : - import StringIO as StringIO -##### +def get_table_list( db, run_name ) : + c = db.cursor() + c.execute("select max(depth) as d, tablename from status where run = ? group by tablename order by d asc",(run_name,)) + table_list = [ x for x in c ] + # table_list contains ( depth, tablename ) + return table_list -def c_d_fn(x,depth) : - if x.depth >= depth : - return - x.depth = depth - depth = depth + 1 - for y in x.children : - c_d_fn(y,depth) - +def get_table( db, run_name, tablename, info_callback ) : + + t = text_table.text_table() + + row = 0 + t.define_column('-') # the command name in column 0 + + c = db.cursor() + c.execute("select distinct host from status where tablename = ? and run = ? order by host asc",(tablename, run_name)) + for host, in c : + t.define_column(host) + t.set_value(row, host, host) + + + c.execute("""select cmd, host, depth, status, start_time, end_time, notes from status + where tablename = ? and run = ? order by depth, cmd asc + """, ( tablename, run_name ) ) + + prev_cmd = None + for x in c : + cmd, host, depth, status, start_time, end_time, notes = x + if cmd != prev_cmd : + row = row + 1 + t.set_value(row, 0, cmd) + prev_cmd = cmd + t.set_value( row, host, info_callback( tablename, cmd, host, status ) ) + + t.pad() + + return t + +def info_callback_status( tablename, cmd, host, status ) : + return status + +def report_text( db, run_name, info_callback = info_callback_status ) : + + s = StringIO.StringIO() + + table_list = get_table_list(db, run_name) + + for depth, tablename in table_list : + s.write("------\n") + s.write(tablename) + s.write('\n') + + t = get_table( db, run_name, tablename, info_callback ) + + s.write( t.get_trac_wiki() ) -def compute_depths(nodes) : - for x in nodes : - x = nodes[x] - x.depth = 0 - x.in_recurse = 0 - x.parents = [ ] - x.children = [ ] - for x in nodes : - x = nodes[x] - for y in x.precursors : - if not x in y.children : - y.children.append(x) - for x in nodes : - c_d_fn(nodes[x],1) - -##### - -def compute_table(nodes) : - # find the depths - compute_depths(nodes) - - # sort the nodes by depth - l = [ x.split(':') for x in nodes ] - l = [ [ x[0] ] + x[1].split('/') for x in l ] - l = [ [ x[1], x[2], x[0] ] for x in l ] - l = sorted(l) - - # table_content is a list of nodes in each table - table_content = { } - - # table_hosts is a list of hosts in each table - table_hosts = { } - - # table_depth is how deep the deepest row of each table is - table_depth = { } - - for x in nodes : - host, table = x.split(':') - table, cmd = table.split('/') - - table_content[table] = table_content.get(table,[]) + [ x ] - - table_hosts [table] = table_hosts .get(table,[]) + [ host ] - - if table_depth.get(table,0) < nodes[x].depth : - table_depth[table] = nodes[x].depth - - for x in table_hosts : - table_hosts[x] = list(set(table_hosts[x])) - - return table_content, table_hosts, table_depth - -##### - -# html of one table - -def html_table(nodes, table, host_list ) : - s=StringIO.StringIO() - - # this is all the nodes in this table - pat = ':%s/' % table - l = [ x for x in nodes if pat in x ] - - # d[x] is the max depth of command x - d = { } - for x in l : - depth = nodes[x].depth - x = x.split('/')[1] - if d.get(x,0) < depth : - d[x] = depth - - # this is the order of the rows of the table - cmd_order = sorted( [ (d[x], x) for x in d ] ) - - # this is the table - s.write( "<table border=1>" ) - - # heading - s.write( "<tr> <td> </td> " ) - for host in host_list : - s.write( "<th>%s</th>" % host ) - s.write( "</tr>\n" ) - - # loop over the commands in the order they appear - for depth, cmd in cmd_order : - s.write( "<tr>\n\t<td>%s/%s</td>\n"%(table,cmd) ) - for host in host_list : - name = host + ':' + table + '/' + cmd - if name in nodes : - s.write( "\t<td class=%%(class!%s)s> %%(text!%s)s </td>\n"%(name,name) ) - else : - s.write( "\t<td class=nothing> . </td>\n" ) - s.write( "</tr>\n" ) - s.write( "</table>" ) return s.getvalue() -class struct(object): - pass +def report_html( db, run_name, info_callback = info_callback_status, hlevel=1 ) : + s = StringIO.StringIO() + s.write('<h%d>%s</h%d>\n'%(hlevel,run_name,hlevel)) -##### + hlevel = hlevel + 1 + + table_list = get_table_list(db, run_name) + + for depth, tablename in table_list : + s.write('<h%d>%s</h%d>\n'%(hlevel,tablename,hlevel)) + t = get_table( db, run_name, tablename, info_callback ) + s.write(t.get_html()) + + return s.getvalue() def main() : - import sqlite3 db = sqlite3.connect('sr.db') + print report_html( db, 'arf2011-08-30 16:52:23.928381' ) + +if __name__ == '__main__' : + main() - c = db.cursor() - c.execute('select run from runs') - for (run,) in c : - - all = [ ] - c1 = db.cursor() - c1.execute('select host, tablename, cmd, depth, status, start_time, end_time, notes from status where run = ? ',(run,) ) - for x in c1 : - n = struct() - n.host, n.tablename, n.cmd, n.depth, n.status, n.start_time, n.ent_time, n.notes = x - all.append(n) - - print "RUN",run - for x in all : - print x.host, x.tablename, x.cmd, x.status - -main() diff --git a/steuermann/run.py b/steuermann/run.py index 7657248..6526e1a 100644 --- a/steuermann/run.py +++ b/steuermann/run.py @@ -160,7 +160,7 @@ class runner(object): def display_procs( self ) : # display currently active child processes - print "procs:", + print "procs:" for x in sorted(self.all_procs) : print " ",x print "" diff --git a/steuermann/run_all.py b/steuermann/run_all.py index 1fe2aa2..59797cb 100644 --- a/steuermann/run_all.py +++ b/steuermann/run_all.py @@ -6,20 +6,36 @@ run everything in a set of command files import time import sys import sqlite3 - -import run - +import os.path import datetime +import run +import report import nodes +try : + import readline +except ImportError : + readline = None + ##### def main() : global xnodes # read all the input files + if readline : + history = os.path.join(os.path.expanduser("~"), ".steuermann_history") + try : + readline.read_history_file(history) + except IOError : + pass + import atexit + atexit.register(readline.write_history_file, history) + + # + di_nodes = nodes.read_file_list( sys.argv[2:] ) xnodes = di_nodes.node_index @@ -30,11 +46,10 @@ def main() : n = sys.argv[1] if n == '-a' : run_all(xnodes, run_name, db) - elif n == '-i' : - run_interactive( xnodes, run_name, db ) else : - print "%s ?"%n + run_interactive( xnodes, run_name, db ) +# def do_flag( xnodes, name, recursive, fn, verbose ) : if verbose : @@ -81,6 +96,8 @@ def cmd_flagging( l, xnodes, func ) : for x in l : do_flag( xnodes, x, recursive, func, 1 ) +# + helpstr = """ report show report want [-r] node declare that we want that node @@ -89,8 +106,9 @@ list -a list node start wait -wr report want/skip values -wd report depth +wr want/skip report +dr depth report +pre node show what must come before a node """ def run_interactive( xnodes, run_name, db) : @@ -106,10 +124,11 @@ def run_interactive( xnodes, run_name, db) : keep_running = 0 while 1 : - print "action?" - l = sys.stdin.readline() - if l == '' : + try : + l = raw_input("smc>") + except EOFError : break + l = l.strip() l = l.split() if len(l) > 0 : @@ -121,13 +140,16 @@ def run_interactive( xnodes, run_name, db) : print helpstr elif n == 'report' : - report( db, run_name ) + print report.report_text( db, run_name ) elif n == 'wr' : - report( db, run_name, info_callback_want ) + print report.report_text( db, run_name, info_callback_want ) + + elif n == 'dr' : + print report.report_text( db, run_name, info_callback_depth ) - elif n == 'wd' : - report( db, run_name, info_callback_depth ) + elif n == 'pre' : + pre_cmd( l[1:], xnodes ) elif n == 'want' : cmd_flagging( l, xnodes, set_want ) @@ -175,6 +197,44 @@ def run_interactive( xnodes, run_name, db) : print "no processes running - some prereq not satisfiable" +# + +def match_all_nodes( l, xnodes ) : + + # all will be the list of all nodes that we want to process + all = [ ] + + # for all the names they said on the command line + for x in l : + + # use wild cards for unspecified prefix parts. i.e. "arf" means "*:*/arf" + x = nodes.normalize_name('*','*',x) + + # find all the nodes that match the pattern + for y in xnodes : + if nodes.wildcard_name( x, y ) : + all.append(y) + + return sorted(all) + +# + +def pre_cmd( l, xnodes ) : + + for x in match_all_nodes( l, xnodes ) : + print "-----" + print x + print_pre(x, xnodes, 1) + + +def print_pre(who, xnodes, depth) : + pre = xnodes[who].predecessors + for x in pre : + x = x.name + print ' '*depth+ x + print_pre( x, xnodes, depth+1) + +# def register_database(db, run, xnodes ) : c = db.cursor() @@ -189,6 +249,8 @@ def register_database(db, run, xnodes ) : db.commit() +# + def run_all(xnodes, run_name, db) : runner = run.runner( xnodes ) @@ -205,6 +267,8 @@ def run_all(xnodes, run_name, db) : if not no_sleep : time.sleep(1) +# + def run_step( runner, xnodes, run_name, db ) : # flag to keep running @@ -299,9 +363,6 @@ def keypress() : ##### -def info_callback_status( tablename, cmd, host, status ) : - return status - def info_callback_want( tablename, cmd, host, status ) : n = xnodes['%s:%s/%s'%(host,tablename,cmd)] s = '' @@ -317,51 +378,6 @@ def info_callback_depth( tablename, cmd, host, status ) : n = xnodes['%s:%s/%s'%(host,tablename,cmd)] return n.depth -def report( db, run_name, info_callback = info_callback_status ) : - import pandokia.text_table as tt - - c = db.cursor() - c.execute("select max(depth) as d, tablename from status where run = ? group by tablename order by d asc",(run_name,)) - table_list = [ x for x in c ] - # table_list contains ( depth, tablename ) - - print """ - -- N = not started - -- S = started, not finished - -- P = prereq not satisfied, so not attempted - -- 0-255 = exit code -""" - - for depth, tablename in table_list : - print "------" - print tablename - t = tt.text_table() - - row = 0 - t.define_column('-') # the command name in column 0 - - c.execute("select distinct host from status where tablename = ? and run = ? order by host asc",(tablename, run_name)) - for host, in c : - t.define_column(host) - t.set_value(row, host, host) - - c.execute("""select cmd, host, depth, status, start_time, end_time, notes from status - where tablename = ? and run = ? order by depth, cmd asc - """, ( tablename, run_name ) ) - - prev_cmd = None - for x in c : - cmd, host, depth, status, start_time, end_time, notes = x - if cmd != prev_cmd : - row = row + 1 - t.set_value(row, 0, cmd) - prev_cmd = cmd - t.set_value( row, host, info_callback( tablename, cmd, host, status ) ) - - t.pad() - - print t.get_trac_wiki() - ##### if __name__ == '__main__' : diff --git a/tests/y.in b/tests/y.in new file mode 100644 index 0000000..60cd109 --- /dev/null +++ b/tests/y.in @@ -0,0 +1,4 @@ +TABLE a HOST host_a + CMD A AFTER C + CMD B AFTER A + CMD C AFTER B @@ -0,0 +1,9 @@ +TABLE a HOST host_a + CMD A RUN "a" AFTER Z + CMD B RUN "b" AFTER A + CMD C RUN "c" AFTER X + CMD D AFTER B + CMD F AFTER * + + CMD E AFTER Z + CMD Z |