checkpoint

git-svn-id: https://svn.stsci.edu/svn/ssb/etal/steuermann/trunk@388 d34015c8-bcbb-4646-8ac8-8ba5febf221d

8 files changed, 307 insertions, 202 deletions

diff --git a/daily.in b/daily.in
new file mode 100644
index 0000000..ae0c843
--- /dev/null
+++ b/daily.in
@@ -0,0 +1,76 @@
+# one approach we might use for ssb builds
+
+TABLE assemble HOST thor
+	CMD svnsync RUN "svnsync"
+		AFTER OPT assemble/irafx_update
+	CMD stsci_python RUN "assemble_stsci_python"
+		AFTER svnsync
+	CMD stsci_iraf RUN "assemble_stsci_iraf"
+		AFTER svnsync
+	CMD astrolib RUN "assemble_astrolib"
+		AFTER svnsync
+	CMD axe RUN "assemble_axe"
+		AFTER svnsync
+	CMD hstcal RUN "assemble_hstcal"
+		AFTER svnsync
+
+# all machines
+TABLE build HOST rhe4-32 rhe4-64 rhe5-64 leopard snow_leopard
+
+	CMD python2.7 RUN "build_stsci_python dev2.7"
+		AFTER *:assemble/stsci_python
+
+	CMD hstcal RUN "build_hstcal"
+		AFTER *:assemble/hstcal
+
+TABLE build HOST rhe-64
+
+	CMD python2.6 RUN "build_stsci_python dev2.6"
+		AFTER *:assemble/stsci_python
+
+	CMD python2.5 RUN "build_stsci_python dev2.5"
+		AFTER *:assemble/stsci_python
+
+
+# 32 bit
+TABLE build32 HOST rhe4-32 leopard
+
+	CMD stsci_iraf RUN "build_stsci_iraf"
+		AFTER *:assemble/stsci_iraf
+
+	CMD axe RUN "build_axe" 
+		AFTER *:assemble/axe
+		AFTER build32/stsci_iraf
+
+# copy 32 bit exe to 64 bit machine
+TABLE build64 HOST rhe4-64 
+	CMD iraf32_hack RUN "iraf32hack rhe4-32"
+
+TABLE build64 HOST rhe5-64 
+	CMD iraf32_hack RUN "iraf32hack rhe4-32"
+
+TABLE build64 HOST snow_leopard
+	CMD iraf32_hack RUN "iraf32hack leopard"
+
+
+# 
+TABLE build_finished HOST rhe4-32 leopard
+	CMD finished2.7 RUN "echo done"
+		AFTER build/python2.7
+		AFTER build/hstcal
+		AFTER build32/stsci_iraf
+		AFTER build32/axe
+
+TABLE build_finished HOST rhe4-64 rhe5-64 snow_leopard
+	CMD finished2.7 RUN "echo done"
+		AFTER build/python2.7
+		AFTER build/hstcal
+		AFTER build64/iraf32_hack
+
+
+#
+TABLE web_updates HOST thor
+	CMD pyraf RUN "exit 1"
+		AFTER *:assemble/svnsync
+	CMD pyfits RUN "exit 1"
+		AFTER *:assemble/svnsync
diff --git a/scripts/steuermann_report.cgi b/scripts/steuermann_report.cgi
new file mode 100644
index 0000000..fb82b89
--- /dev/null
+++ b/scripts/steuermann_report.cgi
@@ -0,0 +1,36 @@
+#! python
+
+import cgi
+import cgitb
+
+cgitb.enable()
+
+form = cgi.FieldStorage(keep_blank_values=1)
+cginame = os.getenv("SCRIPT_NAME")
+
+import sqlite3
+
+if not 'action' in form :
+    print 'content-type: text/html'
+    print ''
+    db = sqlite3.connect('db.sr')
+    c = db.cursor()
+    c.execute('SELECT DISTINCT run FROM status ORDER BY run ASC')
+    for run, in c :
+        print "<a href=%s?action=report&run=%s>%s</a><br>"%(cginame, run, run)
+    return
+
+action = form['action'].value
+
+if action == 'report' :
+    import steuermann.report
+    print 'content-type: text/html'
+    print ''
+    run = form['run'].value
+    print steuermann.report.report_html( db, run )
+    return
+
+print 'content-type: text/html'
+print ''
+print 'no action?'
+
diff --git a/steuermann/nodes.py b/steuermann/nodes.py
index 5e25d8f..dfdb6d7 100644
--- a/steuermann/nodes.py
+++ b/steuermann/nodes.py
@@ -40,16 +40,21 @@ class command_tree(object):
                     if ( fnmatch.fnmatchcase(hl,host ) and 
                          fnmatch.fnmatchcase(tl,table) and 
                          fnmatch.fnmatchcase(cl,cmd  ) ) :
-                        self.connect(x, after, required, pos)
+                        # yes, the wild card matches this one; connect them
+                        # (but don't let a node come before itself because the wild card is too loose)
+                        if x != after :
+                            self.connect(x, after, required, pos)
             else :
                 self.connect(before, after, required, pos)
 
-        # work out the depths of each node
+        # Work out the depths of each node.
+        # Since we are walking the graph, we also detect dependency loops here.
         compute_depths( self.node_index )
         
 
     # make the actual connection between nodes
-    def connect( self, after, before, required, line ) :
+    def connect( self, before, after, required, line ) :
+
         if not after in self.node_index :
             if required :
                 print "error: %s happens after non-existant %s - line %s"%(before,after,line)
@@ -76,7 +81,7 @@ class command_tree(object):
     def add_command_list( self, table, hostlist, command_list ) :
         for host in hostlist :
             this_table = '%s:%s' % ( host, table )
-            for command, script, after, pos in command_list :
+            for command, script, after_clause, pos in command_list :
                 # this happens once for each CMD clause
                 # command is the name of this command
                 # script is the script to run
@@ -92,7 +97,7 @@ class command_tree(object):
                 # create the node
                 self.node_index[command]=node(command, script, nice_pos( current_file_name, pos)  )
 
-                for before_name, required, pos in after :
+                for before_name, required, pos in after_clause :
                     # this happens once for each AFTER clause
                     # before is the name of a predecessor that this one comes after
                     # required is a boolean, whether the predecessor must exist
@@ -237,6 +242,10 @@ def nice_pos( filename, yapps_pos ) :
 
 def c_d_fn(x,depth) :
 
+    if x.in_recursion :
+        print "error: loop detected at",x.name
+        return
+
     # if it is already deeper than where we are now, we can (must)
     # prune the tree walk here.
     if x.depth  >= depth :
@@ -244,9 +253,11 @@ def c_d_fn(x,depth) :
 
     if depth > 100 :
         # bug: proxy for somebody wrote a loop
-        print "error: depth > 100"
+        print "error: depth > 100, node = ",x.name
         return
 
+    x.in_recursion = 1
+
     # assign the depth
     x.depth = depth
 
@@ -255,6 +266,8 @@ def c_d_fn(x,depth) :
     for y in x.successors :
         c_d_fn(y,depth)
 
+    x.in_recursion = 0
+
 def compute_depths(nodes) :
 
     # init everything
@@ -262,6 +275,7 @@ def compute_depths(nodes) :
         x = nodes[x]
         x.depth = 0
         x.successors = [ ]
+        x.in_recursion = 0
 
     # walk the nodes in an arbitrary order; make a list of successors
     for x in nodes :
@@ -275,6 +289,11 @@ def compute_depths(nodes) :
     for x in nodes :
         c_d_fn(nodes[x],1)
 
+    #
+    for x in nodes :
+        x = nodes[x]
+        del x.in_recursion
+
 
 #####
 
@@ -287,7 +306,6 @@ def read_file_list( file_list ) :
     di = command_tree( ) 
     for x in file_list :
         current_file_name = x
-        print x
         sc = specfile.specfileScanner( open(x,'r').read() )
         p = specfile.specfile( scanner=sc, data=di )
         result = exyapps.runtime.wrap_error_reporter( p, 'start' )
diff --git a/steuermann/report.py b/steuermann/report.py
index 1bf4f6a..e1df5ce 100644
--- a/steuermann/report.py
+++ b/steuermann/report.py
@@ -1,146 +1,92 @@
 '''
-prototype of report
-
+Generate reports from the database
+ 
 '''
-import subprocess
+
 import time
 import sys
+import sqlite3
+import pandokia.text_table as text_table
+import StringIO
 
-try :
-    import CStringIO as StringIO
-except ImportError :
-    import StringIO as StringIO
 
-#####
+def get_table_list( db, run_name ) :
+    c = db.cursor()
+    c.execute("select max(depth) as d, tablename from status where run = ? group by tablename order by d asc",(run_name,))
+    table_list = [ x for x in c ]
+        # table_list contains ( depth, tablename )
+    return table_list
 
-def c_d_fn(x,depth) :
-    if x.depth  >= depth :
-        return
-    x.depth = depth
-    depth = depth + 1
-    for y in x.children :
-        c_d_fn(y,depth)
-    
+def get_table( db, run_name, tablename, info_callback ) :
+
+    t = text_table.text_table()
+
+    row = 0
+    t.define_column('-')    # the command name in column 0
+
+    c = db.cursor()
+    c.execute("select distinct host from status where tablename = ? and run = ? order by host asc",(tablename, run_name))
+    for host, in c :
+        t.define_column(host)
+        t.set_value(row, host, host)
+
+
+    c.execute("""select cmd, host, depth, status, start_time, end_time, notes from status
+        where tablename = ? and run = ?  order by depth, cmd asc
+        """, ( tablename, run_name ) )
+
+    prev_cmd = None
+    for x in c :
+        cmd, host, depth, status, start_time, end_time, notes = x
+        if cmd != prev_cmd :
+            row = row + 1
+            t.set_value(row, 0, cmd)
+            prev_cmd = cmd
+        t.set_value( row, host, info_callback( tablename, cmd, host, status ) )
+
+    t.pad()
+
+    return t
+
+def info_callback_status( tablename, cmd, host, status ) :
+    return status
+
+def report_text( db, run_name, info_callback = info_callback_status ) :
+
+    s = StringIO.StringIO()
+
+    table_list = get_table_list(db, run_name)
+
+    for depth, tablename in table_list :
+        s.write("------\n")
+        s.write(tablename)
+        s.write('\n')
+
+        t = get_table( db, run_name, tablename, info_callback )
+
+        s.write( t.get_trac_wiki() )
 
-def compute_depths(nodes) :
-    for x in nodes :
-        x = nodes[x]
-        x.depth = 0
-        x.in_recurse = 0
-        x.parents = [ ]
-        x.children = [ ]
-    for x in nodes :
-        x = nodes[x]
-        for y in x.precursors :
-            if not x in y.children :
-                y.children.append(x)
-    for x in nodes :
-        c_d_fn(nodes[x],1)
-        
-#####
-
-def compute_table(nodes) :
-    # find the depths
-    compute_depths(nodes)
-
-    # sort the nodes by depth
-    l = [ x.split(':') for x in nodes ]
-    l = [ [ x[0] ] + x[1].split('/') for x in l ]
-    l = [ [ x[1], x[2], x[0] ] for x in l ]
-    l = sorted(l)
-
-    # table_content is a list of nodes in each table
-    table_content = { }
-
-    # table_hosts is a list of hosts in each table
-    table_hosts = { }
-
-    # table_depth is how deep the deepest row of each table is
-    table_depth = { }
-
-    for x in nodes  :
-        host, table = x.split(':')
-        table, cmd = table.split('/')
-
-        table_content[table] = table_content.get(table,[]) + [ x ]
-
-        table_hosts  [table] = table_hosts  .get(table,[]) + [ host ]
-
-        if table_depth.get(table,0) < nodes[x].depth :
-            table_depth[table] = nodes[x].depth
-
-    for x in table_hosts :
-        table_hosts[x] = list(set(table_hosts[x]))
-
-    return table_content, table_hosts, table_depth
-
-#####
-
-# html of one table
-
-def html_table(nodes, table, host_list ) :
-    s=StringIO.StringIO()
-
-    # this is all the nodes in this table
-    pat = ':%s/' % table
-    l = [ x for x in nodes if pat in x ]
-
-    # d[x] is the max depth of command x
-    d = { }
-    for x in l :
-        depth = nodes[x].depth
-        x = x.split('/')[1]
-        if d.get(x,0) < depth :
-            d[x] = depth
-
-    # this is the order of the rows of the table
-    cmd_order = sorted( [ (d[x], x) for x in d ] )
-
-    # this is the table
-    s.write( "<table border=1>" )
-
-    # heading
-    s.write( "<tr> <td>&nbsp;</td> " )
-    for host in host_list :
-        s.write( "<th>%s</th>" % host )
-    s.write( "</tr>\n" )
-
-    # loop over the commands in the order they appear
-    for depth, cmd in cmd_order :
-        s.write( "<tr>\n\t<td>%s/%s</td>\n"%(table,cmd) )
-        for host in host_list :
-            name = host + ':' + table + '/' + cmd
-            if name in nodes :
-                s.write( "\t<td class=%%(class!%s)s> %%(text!%s)s </td>\n"%(name,name) )
-            else :
-                s.write( "\t<td class=nothing> . </td>\n" )
-        s.write( "</tr>\n" )
-    s.write( "</table>" )
     return s.getvalue()
 
-class struct(object): 
-    pass
+def report_html( db, run_name, info_callback = info_callback_status, hlevel=1 ) :
+    s = StringIO.StringIO()
+    s.write('<h%d>%s</h%d>\n'%(hlevel,run_name,hlevel))
 
-#####
+    hlevel = hlevel + 1
+    
+    table_list = get_table_list(db, run_name)
+
+    for depth, tablename in table_list :
+        s.write('<h%d>%s</h%d>\n'%(hlevel,tablename,hlevel))
+        t = get_table( db, run_name, tablename, info_callback )
+        s.write(t.get_html())
+
+    return s.getvalue()
 
 def main() :
-    import sqlite3
     db = sqlite3.connect('sr.db')
+    print report_html( db, 'arf2011-08-30 16:52:23.928381' )
+
+if __name__ == '__main__' :
+    main()
 
-    c = db.cursor()
-    c.execute('select run from runs')
-    for (run,) in c :
-
-        all = [ ]
-        c1 = db.cursor()
-        c1.execute('select host, tablename, cmd, depth, status, start_time, end_time, notes from status where run = ? ',(run,) )
-        for x in c1 :
-            n = struct()
-            n.host, n.tablename, n.cmd, n.depth, n.status, n.start_time, n.ent_time, n.notes = x
-            all.append(n)
-
-        print "RUN",run
-        for x in all :
-            print x.host, x.tablename, x.cmd, x.status
-
-main()
diff --git a/steuermann/run.py b/steuermann/run.py
index 7657248..6526e1a 100644
--- a/steuermann/run.py
+++ b/steuermann/run.py
@@ -160,7 +160,7 @@ class runner(object):
 
     def display_procs( self ) :
         # display currently active child processes
-        print "procs:",
+        print "procs:"
         for x in sorted(self.all_procs) :
             print "    ",x
         print ""
diff --git a/steuermann/run_all.py b/steuermann/run_all.py
index 1fe2aa2..59797cb 100644
--- a/steuermann/run_all.py
+++ b/steuermann/run_all.py
@@ -6,20 +6,36 @@ run everything in a set of command files
 import time
 import sys
 import sqlite3
-
-import run
-
+import os.path
 import datetime
 
+import run
+import report
 import nodes
 
 
+try :
+    import readline
+except ImportError :
+    readline = None
+
 #####
 
 def main() :
     global xnodes
     # read all the input files
     
+    if readline :
+        history = os.path.join(os.path.expanduser("~"), ".steuermann_history")
+        try :
+            readline.read_history_file(history)
+        except IOError :
+            pass
+        import atexit
+        atexit.register(readline.write_history_file, history)
+
+    #
+
     di_nodes = nodes.read_file_list( sys.argv[2:] )
 
     xnodes = di_nodes.node_index
@@ -30,11 +46,10 @@ def main() :
     n = sys.argv[1]
     if n == '-a' :
         run_all(xnodes, run_name, db)
-    elif n == '-i' :
-        run_interactive( xnodes, run_name, db )
     else :
-        print "%s ?"%n
+        run_interactive( xnodes, run_name, db )
 
+#
 
 def do_flag( xnodes, name, recursive, fn, verbose ) :
     if verbose :
@@ -81,6 +96,8 @@ def cmd_flagging( l, xnodes, func ) :
     for x in l :
         do_flag( xnodes, x, recursive, func, 1 )
 
+#
+
 helpstr = """
 report              show report 
 want [-r] node      declare that we want that node
@@ -89,8 +106,9 @@ list -a
 list node
 start
 wait
-wr                  report want/skip values
-wd                  report depth
+wr                  want/skip report
+dr                  depth report
+pre node            show what must come before a node
 """
 
 def run_interactive( xnodes, run_name, db) :
@@ -106,10 +124,11 @@ def run_interactive( xnodes, run_name, db) :
     keep_running = 0
 
     while 1 :
-        print "action?"
-        l = sys.stdin.readline()
-        if l == '' :
+        try :
+            l = raw_input("smc>")
+        except EOFError :
             break
+
         l = l.strip()
         l = l.split()
         if len(l) > 0 :
@@ -121,13 +140,16 @@ def run_interactive( xnodes, run_name, db) :
             print helpstr
 
         elif n == 'report' :
-            report( db, run_name )
+            print report.report_text( db, run_name )
 
         elif n == 'wr' :
-            report( db, run_name, info_callback_want )
+            print report.report_text( db, run_name, info_callback_want )
+
+        elif n == 'dr' :
+            print report.report_text( db, run_name, info_callback_depth )
 
-        elif n == 'wd' :
-            report( db, run_name, info_callback_depth )
+        elif n == 'pre' :
+            pre_cmd( l[1:], xnodes )
 
         elif n == 'want' :
             cmd_flagging( l, xnodes, set_want )
@@ -175,6 +197,44 @@ def run_interactive( xnodes, run_name, db) :
                     print "no processes running - some prereq not satisfiable"
         
 
+#
+
+def match_all_nodes( l, xnodes ) :
+
+    # all will be the list of all nodes that we want to process
+    all = [ ]
+
+    # for all the names they said on the command line
+    for x in l :
+
+        # use wild cards for unspecified prefix parts.  i.e. "arf" means "*:*/arf"
+        x = nodes.normalize_name('*','*',x)
+
+        # find all the nodes that match the pattern
+        for y in xnodes :
+            if nodes.wildcard_name( x, y ) :
+                all.append(y)
+
+    return sorted(all)
+
+#
+
+def pre_cmd( l, xnodes ) :
+
+    for x in match_all_nodes( l, xnodes ) :
+        print "-----"
+        print x
+        print_pre(x, xnodes, 1)
+            
+
+def print_pre(who, xnodes, depth) :
+    pre = xnodes[who].predecessors 
+    for x in pre :
+        x = x.name
+        print '  '*depth+ x
+        print_pre( x, xnodes, depth+1)
+
+#
 
 def register_database(db, run, xnodes ) :
     c = db.cursor()
@@ -189,6 +249,8 @@ def register_database(db, run, xnodes ) :
 
     db.commit()
 
+#
+
 def run_all(xnodes, run_name, db) :
 
     runner = run.runner( xnodes )
@@ -205,6 +267,8 @@ def run_all(xnodes, run_name, db) :
         if not no_sleep :
             time.sleep(1)
 
+#
+
 def run_step( runner, xnodes, run_name, db ) :
     
         # flag to keep running 
@@ -299,9 +363,6 @@ def keypress() :
 
 #####
 
-def info_callback_status( tablename, cmd, host, status ) :
-    return status
-
 def info_callback_want( tablename, cmd, host, status ) :
     n = xnodes['%s:%s/%s'%(host,tablename,cmd)]
     s = ''
@@ -317,51 +378,6 @@ def info_callback_depth( tablename, cmd, host, status ) :
     n = xnodes['%s:%s/%s'%(host,tablename,cmd)]
     return n.depth
 
-def report( db, run_name, info_callback = info_callback_status ) :
-    import pandokia.text_table as tt
-
-    c = db.cursor()
-    c.execute("select max(depth) as d, tablename from status where run = ? group by tablename order by d asc",(run_name,))
-    table_list = [ x for x in c ]
-        # table_list contains ( depth, tablename )
-
-    print """
-                -- N = not started
-                -- S = started, not finished
-                -- P = prereq not satisfied, so not attempted
-                -- 0-255 = exit code
-"""
-
-    for depth, tablename in table_list :
-        print "------"
-        print tablename
-        t = tt.text_table()
-
-        row = 0
-        t.define_column('-')    # the command name in column 0
-
-        c.execute("select distinct host from status where tablename = ? and run = ? order by host asc",(tablename, run_name))
-        for host, in c :
-            t.define_column(host)
-            t.set_value(row, host, host)
-
-        c.execute("""select cmd, host, depth, status, start_time, end_time, notes from status
-            where tablename = ? and run = ?  order by depth, cmd asc
-            """, ( tablename, run_name ) )
-
-        prev_cmd = None
-        for x in c :
-            cmd, host, depth, status, start_time, end_time, notes = x
-            if cmd != prev_cmd :
-                row = row + 1
-                t.set_value(row, 0, cmd)
-                prev_cmd = cmd
-            t.set_value( row, host, info_callback( tablename, cmd, host, status ) )
-
-        t.pad()
-
-        print t.get_trac_wiki()
-
 #####
 
 if __name__ == '__main__' :
diff --git a/tests/y.in b/tests/y.in
new file mode 100644
index 0000000..60cd109
--- /dev/null
+++ b/tests/y.in
@@ -0,0 +1,4 @@
+TABLE a HOST host_a
+	CMD A AFTER C
+	CMD B AFTER A
+	CMD C AFTER B
diff --git a/x.in b/x.in
new file mode 100644
index 0000000..a5ab004
--- /dev/null
+++ b/x.in
@@ -0,0 +1,9 @@
+TABLE a HOST host_a
+	CMD A RUN "a" AFTER Z
+	CMD B RUN "b" AFTER A
+	CMD C RUN "c" AFTER X
+	CMD D AFTER B 
+	CMD F AFTER *
+
+	CMD E AFTER Z
+	CMD Z