aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Makefile12
-rw-r--r--daily.sr74
-rw-r--r--note4
-rw-r--r--scripts/smc4
-rw-r--r--setup.py27
-rw-r--r--steuermann/__init__.py1
-rw-r--r--steuermann/db.sql48
-rw-r--r--steuermann/hosts.ini43
-rw-r--r--steuermann/nodes.py303
-rw-r--r--steuermann/report.py146
-rw-r--r--steuermann/run.py206
-rw-r--r--steuermann/run_all.py368
-rw-r--r--steuermann/specfile.exy68
-rw-r--r--tests/misc.py43
14 files changed, 1347 insertions, 0 deletions
diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..bab9fea
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,12 @@
+all: steuermann/specfile.py sr.db
+
+steuermann/specfile.py: steuermann/specfile.exy
+ exyapps steuermann/specfile.exy
+
+sr.db: steuermann/db.sql
+ rm -f sr.db
+ sqlite3 sr.db < steuermann/db.sql
+
+clean:
+ rm -f sr.db
+
diff --git a/daily.sr b/daily.sr
new file mode 100644
index 0000000..13e856e
--- /dev/null
+++ b/daily.sr
@@ -0,0 +1,74 @@
+TABLE assemble HOST thor
+ CMD svnsync RUN "svnsync"
+ AFTER OPT assemble/irafx_update
+ CMD stsci_python RUN "assemble_stsci_python"
+ AFTER svnsync
+ CMD stsci_iraf RUN "assemble_stsci_iraf"
+ AFTER svnsync
+ CMD astrolib RUN "assemble_astrolib"
+ AFTER svnsync
+ CMD axe RUN "assemble_axe"
+ AFTER svnsync
+ CMD hstcal RUN "assemble_hstcal"
+ AFTER svnsync
+
+# all machines
+TABLE build HOST rhe4-32 rhe4-64 rhe5-64 leopard snow_leopard
+
+ CMD python2.7 RUN "build_stsci_python dev2.7"
+ AFTER *:assemble/stsci_python
+
+ CMD hstcal RUN "build_hstcal"
+ AFTER *:assemble/hstcal
+
+TABLE build HOST rhe-64
+
+ CMD python2.6 RUN "build_stsci_python dev2.6"
+ AFTER *:assemble/stsci_python
+
+ CMD python2.5 RUN "build_stsci_python dev2.5"
+ AFTER *:assemble/stsci_python
+
+
+# 32 bit
+TABLE build32 HOST rhe4-32 leopard
+
+ CMD stsci_iraf RUN "build_stsci_iraf"
+ AFTER *:assemble/stsci_iraf
+
+ CMD axe RUN "build_axe"
+ AFTER *:assemble/axe
+ AFTER build32/stsci_iraf
+
+# copy 32 bit exe to 64 bit machine
+TABLE build64 HOST rhe4-64
+ CMD iraf32_hack RUN "iraf32hack rhe4-32"
+
+TABLE build64 HOST rhe5-64
+ CMD iraf32_hack RUN "iraf32hack rhe4-32"
+
+TABLE build64 HOST snow_leopard
+ CMD iraf32_hack RUN "iraf32hack leopard"
+
+
+#
+TABLE build_finished HOST rhe4-32 leopard
+ CMD finished2.7 RUN "echo done"
+ AFTER build/python2.7
+ AFTER build/hstcal
+ AFTER build32/stsci_iraf
+ AFTER build32/axe
+
+TABLE build_finished HOST rhe4-64 rhe5-64 snow_leopard
+ CMD finished2.7 RUN "echo done"
+ AFTER build/python2.7
+ AFTER build/hstcal
+ AFTER build64/iraf32_hack
+
+
+#
+TABLE web_updates HOST thor
+ CMD pyraf RUN "exit 1"
+ AFTER *:assemble/svnsync
+ CMD pyfits RUN "exit 1"
+ AFTER *:assemble/svnsync
diff --git a/note b/note
new file mode 100644
index 0000000..c185696
--- /dev/null
+++ b/note
@@ -0,0 +1,4 @@
+
+http://stackoverflow.com/questions/2408560/python-nonblocking-console-input
+
+
diff --git a/scripts/smc b/scripts/smc
new file mode 100644
index 0000000..3937855
--- /dev/null
+++ b/scripts/smc
@@ -0,0 +1,4 @@
+#! python
+import steuermann.run_all as run_all
+
+run_all.main()
diff --git a/setup.py b/setup.py
new file mode 100644
index 0000000..42d5f3a
--- /dev/null
+++ b/setup.py
@@ -0,0 +1,27 @@
+import distutils.core
+
+f=open('steuermann/__init__.py','r')
+for x in f :
+ if x.startswith('__version__') :
+ version = x.split("'")[1]
+ break
+f.close()
+
+print version
+
+args = {
+ 'version' : "2.0dev",
+ 'description' : "Steuermann Continuous Integration Control System",
+ 'author' : "Mark Sienkiewicz",
+ 'scripts' : ['scripts/smc'],
+ 'package_dir' : { 'steuermann' : 'steuermann', },
+ 'url' : 'https://svn.stsci.edu/trac/ssb/etal/wiki/Steuermann',
+ 'license': 'BSD',
+ 'packages': [ 'steuermann' ],
+ 'package_data': { 'steuermann' : [ '*.sql', '*.ini', ], }
+}
+
+d = distutils.core.setup(
+ **args
+)
+
diff --git a/steuermann/__init__.py b/steuermann/__init__.py
new file mode 100644
index 0000000..0e77cbd
--- /dev/null
+++ b/steuermann/__init__.py
@@ -0,0 +1 @@
+__version__ = '2.0dev'
diff --git a/steuermann/db.sql b/steuermann/db.sql
new file mode 100644
index 0000000..664f5df
--- /dev/null
+++ b/steuermann/db.sql
@@ -0,0 +1,48 @@
+-- The table 'status' contains a record for each command in a run.
+-- Before we start running anything, we insert a record for every
+-- command in the test run. The initial status is 'S'.
+
+CREATE TABLE status (
+ run VARCHAR(100),
+ -- name of this run
+
+ host VARCHAR,
+ tablename VARCHAR,
+ cmd VARCHAR,
+ -- name of the command (node)
+
+ depth INTEGER,
+ -- depth in the tree of this node (used to create report tables)
+
+ status VARCHAR(5),
+ -- N = not started
+ -- S = started, not finished
+ -- P = prereq not satisfied, so not attempted
+ -- 0-255 = exit code
+
+ start_time VARCHAR(30),
+ end_time VARCHAR(30),
+ -- times initially blank
+ -- YYYY-MM-DD HH:MM:SS.SSS
+ -- (space for resolution to nanosecond is a bit extreme)
+
+ -- a log file name is implicit in the run/host/tablename/cmd tuple
+
+ notes VARCHAR(1000),
+ -- notes reported by the script
+
+ FOREIGN KEY(run) REFERENCES runs(run)
+ -- run name has to be in the run table
+ );
+
+
+create unique index idx_status_1 on status ( run, host, tablename, cmd );
+
+
+-- table lists all run names in the system
+CREATE TABLE runs (
+ run VARCHAR(100)
+ );
+
+CREATE UNIQUE INDEX idx_runs_run ON runs(run);
+
diff --git a/steuermann/hosts.ini b/steuermann/hosts.ini
new file mode 100644
index 0000000..61231df
--- /dev/null
+++ b/steuermann/hosts.ini
@@ -0,0 +1,43 @@
+; semicolon starts a comment in INI files
+; If your data value contains a ";" make sure there is no space before it
+
+; There is a section for each [machinename]. In that section, you
+; should define
+; run=some command to run scripts
+; It is a list of argv values to run the command. In each item of
+; the list, you can use %(n)s to replace with various values:
+; %(script)s -
+; %(host)s -
+; %(table)s -
+; %(cmd)s -
+; %(foo)s - if you include a foo= line
+;
+[arzach]
+workdir=/arzach/data1/sienkiew
+
+[thor]
+workdir=/thor/data2/sienkiew
+
+[herbert]
+workdir=/herbert/data1/sienkiew
+
+[jwcalibdev]
+workdir=/data1/sienkiew/ur_work
+
+; There is a section [ALL] that is used with every machine name
+[ALL]
+run=[ 'ssh', '-q', '%(host)s', 'cd %(workdir)s; %(script)s' ]
+
+[host_a]
+run=[ 'sleep', '3' ]
+
+[host_b]
+run=[ 'sleep', '3' ]
+
+[host_c]
+run=[ 'sleep', '3' ]
+
+[host_d]
+run=[ 'sleep', '3' ]
+
+
diff --git a/steuermann/nodes.py b/steuermann/nodes.py
new file mode 100644
index 0000000..5e25d8f
--- /dev/null
+++ b/steuermann/nodes.py
@@ -0,0 +1,303 @@
+'''
+Stuff related to the tree structure of the command set
+'''
+
+import fnmatch
+import exyapps.runtime
+
+
+#####
+#####
+
+class command_tree(object):
+
+ # a dict that maps currently known node names to node objects
+ node_index = None
+
+ # This is a list if (before, after) showing names of node orders;
+ # the individual nodes do not store this information while the
+ # parsing is happening.
+ node_order = None
+
+ # call init() once before parsing, then call parse for each file, then call finish()
+ def __init__( self ) :
+ self.node_index = { }
+ self.node_order = [ ]
+
+ def finish( self ) :
+ # This links up the nodes according to the (before, after) information
+ # in node_order. before and after are fully qualified node names.
+ # We do this at the end so that
+ # - we can have forward references
+ # - we can use wild cards in our AFTER clauses
+
+ # before is the predecessor, after comes AFTER the predecessor
+ for before, after, required, pos in self.node_order :
+ if ( '*' in before ) or ( '?' in before ) or ( '[' in before ):
+ host, table, cmd = crack_name(before)
+ for x in self.node_index :
+ hl, tl, cl = crack_name(x)
+ if ( fnmatch.fnmatchcase(hl,host ) and
+ fnmatch.fnmatchcase(tl,table) and
+ fnmatch.fnmatchcase(cl,cmd ) ) :
+ self.connect(x, after, required, pos)
+ else :
+ self.connect(before, after, required, pos)
+
+ # work out the depths of each node
+ compute_depths( self.node_index )
+
+
+ # make the actual connection between nodes
+ def connect( self, after, before, required, line ) :
+ if not after in self.node_index :
+ if required :
+ print "error: %s happens after non-existant %s - line %s"%(before,after,line)
+ return
+
+ if not before in self.node_index :
+ print "error: before node %s does not exist %s"%(before,line)
+ return
+
+ if not after in self.node_index :
+ print "error: after node %s does not exist %s"%(after,line)
+ return
+
+ # tell the after node that the other one comes first
+
+ self.node_index[after].predecessors.append(self.node_index[before])
+
+ # tell the after node about the before node and note that
+ # the before node is not done yet.
+ self.node_index[after].released[before] = False
+
+
+ # create a set of nodes for a particular command - called from within the parser
+ def add_command_list( self, table, hostlist, command_list ) :
+ for host in hostlist :
+ this_table = '%s:%s' % ( host, table )
+ for command, script, after, pos in command_list :
+ # this happens once for each CMD clause
+ # command is the name of this command
+ # script is the script to run
+ # after is a list of AFTER clauses
+ # pos is where in this file this command was defined
+
+ command = normalize_name( host, table, command )
+
+ if command in self.node_index :
+ # bug: should be error
+ print "# warning: %s already used on line %s"%(command,self.node_index[command].input_line)
+
+ # create the node
+ self.node_index[command]=node(command, script, nice_pos( current_file_name, pos) )
+
+ for before_name, required, pos in after :
+ # this happens once for each AFTER clause
+ # before is the name of a predecessor that this one comes after
+ # required is a boolean, whether the predecessor must exist
+ # pos is where in the file the AFTER clause begins
+
+ before_name = normalize_name( host, table, before_name )
+
+ # list of ( before, after, required, pos )
+ self.node_order.append( (before_name, command, required, pos) )
+
+
+
+#####
+
+# crack open host:table/cmd
+def crack_name(name) :
+ t = name.split(':')
+ host = t[0]
+ t = t[1].split('/')
+ table = t[0]
+ cmd = t[1]
+ return (host, table, cmd)
+
+#####
+
+def join_name( host, table, cmd ) :
+ return '%s:%s/%s'%(host,table,cmd)
+
+#####
+
+def normalize_name( host, table, name ) :
+
+ if not '/' in name :
+ name = table + '/' + name
+
+ if name.startswith('.:') :
+ name = name[2:]
+
+ if not ':' in name :
+ name = host + ':' + name
+ # print "## return %s"%name
+ return name
+
+#####
+
+
+_wildcard_cache = ( None, None )
+
+def wildcard_name( wild, name ) :
+ global _wildcard_cache
+
+ if wild != _wildcard_cache :
+ host, table, cmd = crack_name(wild)
+ _wildcard_cache = ( name, ( host, table, cmd ) )
+
+ host, table, cmd = _wildcard_cache[1]
+
+ hl, tl, cl = crack_name(name)
+
+ return ( fnmatch.fnmatchcase(hl,host ) and
+ fnmatch.fnmatchcase(tl,table) and
+ fnmatch.fnmatchcase(cl,cmd ) )
+
+
+#####
+
+
+#####
+
+
+# a node object for each command instance that will be run. The name is
+# "host:table/command". The shell command to execute on the target host
+# is "script".
+#
+# predecessors[] is a list of everything that this node must definitely come after.
+#
+# released is a dict indexed by each of the "before" nodes; the value is
+# true if the before node is finished running.
+#
+class node(object) :
+ def __init__(self, name, script, input_line) :
+ # the fully qualified name of the node
+ self.name = name
+
+ # the command script that this node runs
+ self.script = script
+
+ # what line of the input file specified this node; this is
+ # a string of the form "foo.bar 123"
+ self.input_line = input_line
+
+ # crack open host:table/cmd
+ self.host, self.table, self.cmd = crack_name(name)
+
+ # this command runs after every node in this list
+ self.predecessors = [ ]
+
+ # this is a dict of everything that comes before us
+ # The key is the name of the other node.
+ # The value is true/false for whether the other node
+ # is finished running.
+ self.released = { }
+
+ # These flags are 1 or 0 so we can sum() them
+ self.finished = 0
+ self.running = 0
+
+#####
+
+# debug - make a string representation of all the nodes
+
+def show_nodes( node_index ) :
+ import cStringIO as StringIO
+ s = StringIO.StringIO()
+ for x in sorted( [ x for x in node_index ] ) :
+ x = node_index[x]
+
+ # show the node name and the command it runs
+ s.write( "NODE %s %s %s\n"%(x.name, x.script, x.input_line) )
+
+ # show each node that this one comes After, and show the flag
+ # whether the previous node has run yet.
+ for y in x.predecessors :
+ s.write( " AFTER %s\n"%y.name )
+
+ return s.getvalue()
+
+#####
+
+def nice_pos( filename, yapps_pos ) :
+ # filename is the file name to use
+ # yapps_pos is a tuple of (file, line, col) except that file
+ # is a file-like object with no traceability to an actual file1
+ return "%s:%s col %s"%(filename, yapps_pos[1], yapps_pos[2])
+
+#####
+
+
+#####
+#
+# calculating depths of a node tree
+
+def c_d_fn(x,depth) :
+
+ # if it is already deeper than where we are now, we can (must)
+ # prune the tree walk here.
+ if x.depth >= depth :
+ return
+
+ if depth > 100 :
+ # bug: proxy for somebody wrote a loop
+ print "error: depth > 100"
+ return
+
+ # assign the depth
+ x.depth = depth
+
+ # make all the successors one deeper
+ depth = depth + 1
+ for y in x.successors :
+ c_d_fn(y,depth)
+
+def compute_depths(nodes) :
+
+ # init everything
+ for x in nodes :
+ x = nodes[x]
+ x.depth = 0
+ x.successors = [ ]
+
+ # walk the nodes in an arbitrary order; make a list of successors
+ for x in nodes :
+ x = nodes[x]
+ for y in x.predecessors :
+ if not x in y.successors :
+ y.successors.append(x)
+
+ # recursively walk down the tree assigning depth values, starting
+ # with depth=1 for the highest level
+ for x in nodes :
+ c_d_fn(nodes[x],1)
+
+
+#####
+
+import specfile
+
+current_file_name = None
+
+def read_file_list( file_list ) :
+ global current_file_name
+ di = command_tree( )
+ for x in file_list :
+ current_file_name = x
+ print x
+ sc = specfile.specfileScanner( open(x,'r').read() )
+ p = specfile.specfile( scanner=sc, data=di )
+ result = exyapps.runtime.wrap_error_reporter( p, 'start' )
+ di.finish()
+ return di
+
+if __name__=='__main__':
+ import sys
+ n = read_file_list( sys.argv[1:] )
+ print show_nodes(n.node_index)
+
+
+
diff --git a/steuermann/report.py b/steuermann/report.py
new file mode 100644
index 0000000..1bf4f6a
--- /dev/null
+++ b/steuermann/report.py
@@ -0,0 +1,146 @@
+'''
+prototype of report
+
+'''
+import subprocess
+import time
+import sys
+
+try :
+ import CStringIO as StringIO
+except ImportError :
+ import StringIO as StringIO
+
+#####
+
+def c_d_fn(x,depth) :
+ if x.depth >= depth :
+ return
+ x.depth = depth
+ depth = depth + 1
+ for y in x.children :
+ c_d_fn(y,depth)
+
+
+def compute_depths(nodes) :
+ for x in nodes :
+ x = nodes[x]
+ x.depth = 0
+ x.in_recurse = 0
+ x.parents = [ ]
+ x.children = [ ]
+ for x in nodes :
+ x = nodes[x]
+ for y in x.precursors :
+ if not x in y.children :
+ y.children.append(x)
+ for x in nodes :
+ c_d_fn(nodes[x],1)
+
+#####
+
+def compute_table(nodes) :
+ # find the depths
+ compute_depths(nodes)
+
+ # sort the nodes by depth
+ l = [ x.split(':') for x in nodes ]
+ l = [ [ x[0] ] + x[1].split('/') for x in l ]
+ l = [ [ x[1], x[2], x[0] ] for x in l ]
+ l = sorted(l)
+
+ # table_content is a list of nodes in each table
+ table_content = { }
+
+ # table_hosts is a list of hosts in each table
+ table_hosts = { }
+
+ # table_depth is how deep the deepest row of each table is
+ table_depth = { }
+
+ for x in nodes :
+ host, table = x.split(':')
+ table, cmd = table.split('/')
+
+ table_content[table] = table_content.get(table,[]) + [ x ]
+
+ table_hosts [table] = table_hosts .get(table,[]) + [ host ]
+
+ if table_depth.get(table,0) < nodes[x].depth :
+ table_depth[table] = nodes[x].depth
+
+ for x in table_hosts :
+ table_hosts[x] = list(set(table_hosts[x]))
+
+ return table_content, table_hosts, table_depth
+
+#####
+
+# html of one table
+
+def html_table(nodes, table, host_list ) :
+ s=StringIO.StringIO()
+
+ # this is all the nodes in this table
+ pat = ':%s/' % table
+ l = [ x for x in nodes if pat in x ]
+
+ # d[x] is the max depth of command x
+ d = { }
+ for x in l :
+ depth = nodes[x].depth
+ x = x.split('/')[1]
+ if d.get(x,0) < depth :
+ d[x] = depth
+
+ # this is the order of the rows of the table
+ cmd_order = sorted( [ (d[x], x) for x in d ] )
+
+ # this is the table
+ s.write( "<table border=1>" )
+
+ # heading
+ s.write( "<tr> <td>&nbsp;</td> " )
+ for host in host_list :
+ s.write( "<th>%s</th>" % host )
+ s.write( "</tr>\n" )
+
+ # loop over the commands in the order they appear
+ for depth, cmd in cmd_order :
+ s.write( "<tr>\n\t<td>%s/%s</td>\n"%(table,cmd) )
+ for host in host_list :
+ name = host + ':' + table + '/' + cmd
+ if name in nodes :
+ s.write( "\t<td class=%%(class!%s)s> %%(text!%s)s </td>\n"%(name,name) )
+ else :
+ s.write( "\t<td class=nothing> . </td>\n" )
+ s.write( "</tr>\n" )
+ s.write( "</table>" )
+ return s.getvalue()
+
+class struct(object):
+ pass
+
+#####
+
+def main() :
+ import sqlite3
+ db = sqlite3.connect('sr.db')
+
+ c = db.cursor()
+ c.execute('select run from runs')
+ for (run,) in c :
+
+ all = [ ]
+ c1 = db.cursor()
+ c1.execute('select host, tablename, cmd, depth, status, start_time, end_time, notes from status where run = ? ',(run,) )
+ for x in c1 :
+ n = struct()
+ n.host, n.tablename, n.cmd, n.depth, n.status, n.start_time, n.ent_time, n.notes = x
+ all.append(n)
+
+ print "RUN",run
+ for x in all :
+ print x.host, x.tablename, x.cmd, x.status
+
+main()
diff --git a/steuermann/run.py b/steuermann/run.py
new file mode 100644
index 0000000..7657248
--- /dev/null
+++ b/steuermann/run.py
@@ -0,0 +1,206 @@
+'''
+run processes asynchronously on various machines, with a callback
+on process exit.
+'''
+
+import subprocess
+import time
+import datetime
+import os
+
+import ConfigParser
+
+debug=0
+
+#####
+
+class struct :
+ pass
+
+
+#####
+
+class runner(object):
+
+ # dict of all current running processes, indexed by node name
+ all_procs = { }
+
+ # index of nodes
+ node_index = { }
+
+ # info about hosts we can run on
+ host_info = { }
+
+ # dir where we write our logs
+ logdir = 'logs'
+
+ #####
+ #
+
+ def __init__( self, nodes ) :
+ self.node_index = nodes
+ self.load_host_info()
+
+ #####
+ # start a process
+
+ def run( self, node, run_name ):
+ if debug :
+ print "run",node.name
+ node.running = 1
+ if debug :
+ print "....%s:%s/%s\n"%(node.host, node.table, node.cmd)
+
+ try :
+ args = self.host_info[node.host]
+ except :
+ print "ERROR: do not know how to run on %s"%node.host
+ raise
+
+ run = args['run']
+
+ args = args.copy()
+ args.update(
+ script=node.script,
+ host=node.host,
+ table=node.table,
+ cmd=node.cmd
+ )
+
+ if debug :
+ print "ARGS"
+ for x in sorted([x for x in args]) :
+ print '%s=%s'%(x,args[x])
+
+ run = [ x % args for x in run ]
+
+ if debug :
+ print "RUN",run
+
+ # make sure the log directory is there
+ logdir= self.logdir + "/%s"%run_name
+ try :
+ os.makedirs(logdir)
+ except OSError:
+ pass
+
+ # create a name for the log file, but do not use / in the name
+ logfile_name = "%s/%s.log"%( logdir, node.name.replace('/','+') )
+
+ # open the log file, write initial notes
+ logfile=open(logfile_name,"w")
+ logfile.write('%s %s\n'%(datetime.datetime.now(),run))
+ logfile.flush()
+
+ # start running the process
+ p = subprocess.Popen(args=run,
+ stdout=logfile,
+ stderr=subprocess.STDOUT,
+ shell=False, close_fds=True)
+
+ # remember the popen object for the process; remember the open log file
+ n = struct()
+ n.proc = p
+ n.logfile = logfile
+ n.logfile_name = logfile_name
+
+ # remember the process is running
+ self.all_procs[node.name] = n
+
+ #####
+ # callback when a node finishes
+
+ def finish( self, node_name, status):
+ # note the termination of the process at the end of the log file
+ logfile = self.all_procs[node_name].logfile
+ logfile.seek(0,2) # end of file
+ logfile.write('\n%s exit=%s\n'%(datetime.datetime.now(),status))
+ logfile.close()
+
+ # note the completion of the command
+ node = self.node_index[node_name]
+ if debug :
+ print "finish",node.name
+ node.running = 0
+ node.finished = 1
+ node.exit_status = status
+
+ #####
+
+ # poll for exited child processes - this whole thing could could
+ # be event driven, but I don't care to work out the details right
+ # now.
+
+ def poll( self ) :
+
+ # look at all active processes
+ for name in self.all_procs :
+
+ # see if name has finished
+ p = self.all_procs[name].proc
+ n = p.poll()
+ if n is not None :
+
+ # marke the node finished
+ self.finish(name,n)
+
+ #
+ status = p.returncode
+
+ # remove it from the list of pending processes
+ del self.all_procs[name]
+
+ # Return the identity of the exited process.
+ # There may be more, but we will come back and poll again.
+ return ( name, status )
+
+ return None
+
+ #####
+
+ def display_procs( self ) :
+ # display currently active child processes
+ print "procs:",
+ for x in sorted(self.all_procs) :
+ print " ",x
+ print ""
+
+ #####
+
+
+ def _host_get_names( self, cfg, section, d ) :
+ # pick all the variables out of this section
+ for name, value in cfg.items(section) :
+ if name == 'run' :
+ # run is a list
+ d[name] = eval(value)
+ else :
+ # everything else is plain text
+ d[name] = value
+
+ def load_host_info( self, filename=None ) :
+ self.host_info = { }
+
+ # read the config file
+ if filename is None :
+ filename = os.path.dirname(__file__) + '/hosts.ini'
+ cfg = ConfigParser.RawConfigParser()
+ cfg.read(filename)
+
+ # this dict holds the set of values that are defined as
+ # applying to all hosts
+ all_dict = { }
+ self._host_get_names(cfg, 'ALL', all_dict)
+
+ # for all the sections (except ALL) get the names from that section
+ for x in cfg.sections() :
+ if x == 'ALL' :
+ continue
+
+ d = all_dict.copy()
+ self._host_get_names(cfg, x, d)
+ self.host_info[x] = d
+
+ del cfg
+
+ #####
diff --git a/steuermann/run_all.py b/steuermann/run_all.py
new file mode 100644
index 0000000..1fe2aa2
--- /dev/null
+++ b/steuermann/run_all.py
@@ -0,0 +1,368 @@
+'''
+run everything in a set of command files
+
+'''
+
+import time
+import sys
+import sqlite3
+
+import run
+
+import datetime
+
+import nodes
+
+
+#####
+
+def main() :
+ global xnodes
+ # read all the input files
+
+ di_nodes = nodes.read_file_list( sys.argv[2:] )
+
+ xnodes = di_nodes.node_index
+ run_name = 'arf' + str(datetime.datetime.now())
+ db = sqlite3.connect('sr.db')
+ register_database(db, run_name, xnodes)
+
+ n = sys.argv[1]
+ if n == '-a' :
+ run_all(xnodes, run_name, db)
+ elif n == '-i' :
+ run_interactive( xnodes, run_name, db )
+ else :
+ print "%s ?"%n
+
+
+def do_flag( xnodes, name, recursive, fn, verbose ) :
+ if verbose :
+ verbose = verbose + 1
+ if not ':' in name :
+ name = '*:' + name
+ if ( '*' in name ) or ( '?' in name ) or ( '[' in name ) :
+ if verbose :
+ print ' '*verbose, "wild",name
+ for x in xnodes :
+ if nodes.wildcard_name( name, x ) :
+ if verbose :
+ print ' '*verbose, "match",x
+ do_flag( xnodes, x, recursive, fn, verbose )
+ elif name in xnodes :
+ if verbose :
+ print ' '*verbose, "found",name
+ fn(xnodes[name])
+ if recursive :
+ for y in xnodes[name].predecessors :
+ do_flag( xnodes, y.name, recursive, fn, verbose )
+ else :
+ if verbose :
+ print ' '*verbose, "not in list", name
+ raise Exception()
+
+def set_want( node ) :
+ node.wanted = 1
+ node.skip = 0
+
+def set_skip( node ) :
+ node.wanted = 0
+ node.skip = 1
+
+
+def cmd_flagging( l, xnodes, func ) :
+ if l[1] == '-r' :
+ recursive = 1
+ l = l[2:]
+ else :
+ recursive = 0
+ l = l[1:]
+
+ for x in l :
+ do_flag( xnodes, x, recursive, func, 1 )
+
+helpstr = """
+report show report
+want [-r] node declare that we want that node
+skip [-r] node skip this node
+list -a
+list node
+start
+wait
+wr report want/skip values
+wd report depth
+"""
+
+def run_interactive( xnodes, run_name, db) :
+
+ runner = run.runner( xnodes )
+
+ for x in xnodes :
+ xnodes[x].finished = 0
+ xnodes[x].running = 0
+ xnodes[x].wanted = 0
+ xnodes[x].skip = 0
+
+ keep_running = 0
+
+ while 1 :
+ print "action?"
+ l = sys.stdin.readline()
+ if l == '' :
+ break
+ l = l.strip()
+ l = l.split()
+ if len(l) > 0 :
+ n = l[0]
+ else :
+ n = ''
+
+ if n == '?' :
+ print helpstr
+
+ elif n == 'report' :
+ report( db, run_name )
+
+ elif n == 'wr' :
+ report( db, run_name, info_callback_want )
+
+ elif n == 'wd' :
+ report( db, run_name, info_callback_depth )
+
+ elif n == 'want' :
+ cmd_flagging( l, xnodes, set_want )
+
+ elif n == 'skip' :
+ cmd_flagging( l, xnodes, set_skip )
+
+ elif n == 'list' :
+ print_all = '-a' in l
+ l = sorted ( [ x for x in xnodes ] )
+ print "w f s name"
+ for x in l :
+ print xnodes[x].wanted, xnodes[x].finished, xnodes[x].skip, x
+ if print_all :
+ print " AFTER", ' '.join([ a.name for a in xnodes[x].predecessors ])
+
+ elif n == 'start' :
+ keep_running = 1
+
+ elif n == 'wait' :
+ while 1 :
+ ( keep_running, no_sleep ) = run_step( runner, xnodes, run_name, db )
+ if not keep_running :
+ break
+ if not no_sleep :
+ time.sleep(1)
+ if keypress() :
+ print "wait interrupted (processes continue)"
+ break
+ print "wait finished"
+
+ if keep_running :
+ print "run step"
+ ( keep_running, no_sleep ) = run_step( runner, xnodes, run_name, db )
+
+ if len(runner.all_procs) == 0 :
+ # give it a chance to start another
+ ( keep_running, no_sleep ) = run_step( runner, xnodes, run_name, db )
+
+ if not keep_running :
+ print 'all done'
+
+ else :
+ if len(runner.all_procs) == 0 :
+ print "no processes running - some prereq not satisfiable"
+
+
+
+def register_database(db, run, xnodes ) :
+ c = db.cursor()
+ c.execute('INSERT INTO runs ( run ) VALUES ( ? )', ( run, ) )
+
+ c = db.cursor()
+ for x in xnodes :
+ host, tablename, cmd = nodes.crack_name(x)
+ depth = xnodes[x].depth
+ c.execute("INSERT INTO status ( run, host, tablename, cmd, depth, status ) VALUES "
+ "( ?, ?, ?, ?, ?, 'N' )", ( run, host, tablename, cmd, depth ) )
+
+ db.commit()
+
+def run_all(xnodes, run_name, db) :
+
+ runner = run.runner( xnodes )
+
+ for x in xnodes :
+ xnodes[x].finished = 0
+ xnodes[x].running = 0
+ xnodes[x].wanted = 1
+
+ while 1 :
+ ( keep_running, no_sleep ) = run_step( runner, xnodes, run_name, db )
+ if not keep_running :
+ break
+ if not no_sleep :
+ time.sleep(1)
+
+def run_step( runner, xnodes, run_name, db ) :
+
+ # flag to keep running
+ keep_running = 0
+
+ # flag to suppress brief sleep at end of loop
+ no_sleep = 0
+
+ # Loop, polling for work to do, or for finishing processes
+ print "loop"
+ for x_name in xnodes :
+ x=xnodes[x_name]
+
+ # skip nodes that we do not need to consider running because
+
+ # - we explicitly ask to skip it; also mark it finished
+ # so that things that come after can run
+ if x.skip :
+ x.finished = 1
+ continue
+
+ # - it is not wanted
+ if not x.wanted :
+ continue
+
+ # - it is already finished
+ if x.finished :
+ continue
+
+ # - we are already running it
+ if x.running :
+ keep_running=1
+ continue
+
+ # ok, if we are here, we found a node that we want to run
+
+ # if there is a node we need to run, we need to come back through the loop
+ # (bug: are we sure there is not a deadlock caused by mutual dependencies? if that happens, it can never run.)
+ keep_running = 1
+
+ # count how many of the predecessors are finished
+ released = sum( [ xnodes[r].finished for r in x.released ])
+
+ # if the number of predecessors finished is the number
+ # of predecessors, we can run this one
+ if released == len(x.released) :
+ host, table, cmd = nodes.crack_name(x_name)
+ print "RUN", x_name
+
+ db.execute("UPDATE status SET start_time = ?, status = 'S' WHERE ( run = ? AND host = ? AND tablename = ? AND cmd = ? )",
+ ( str(datetime.datetime.now()), run_name, host, table, cmd ) )
+ db.commit()
+ runner.run(x, run_name)
+
+ # if anything has exited, we process it and update the status in the database
+ while 1 :
+ who_exited = runner.poll()
+ if not who_exited :
+ break
+
+ # yes, something exited - no sleep, and keep running
+ no_sleep = 1
+ keep_running = 1
+
+ # note who and log it
+ x_host, x_table, x_cmd = nodes.crack_name(who_exited[0])
+
+ xnodes[who_exited[0]].wanted = 0
+
+ db.execute("UPDATE status SET end_time = ?, status = ? WHERE ( run = ? AND host = ? AND tablename = ? AND cmd = ? )",
+ ( str(datetime.datetime.now()), who_exited[1], run_name, x_host, x_table, x_cmd ) )
+
+
+ runner.display_procs()
+
+ return ( keep_running, no_sleep )
+
+#####
+
+ms_windows = 0
+
+if ms_windows :
+ import msvcrt
+else :
+ import select
+
+def keypress() :
+ if ms_windows :
+ return msvcrt.kbhit()
+ else :
+ return select.select([sys.stdin], [], [], 0) == ([sys.stdin], [], [])
+
+#####
+
+def info_callback_status( tablename, cmd, host, status ) :
+ return status
+
+def info_callback_want( tablename, cmd, host, status ) :
+ n = xnodes['%s:%s/%s'%(host,tablename,cmd)]
+ s = ''
+ if n.skip :
+ s = s + 'S'
+ if n.wanted :
+ s = s + 'W'
+ if s == '' :
+ s = '-'
+ return s
+
+def info_callback_depth( tablename, cmd, host, status ) :
+ n = xnodes['%s:%s/%s'%(host,tablename,cmd)]
+ return n.depth
+
+def report( db, run_name, info_callback = info_callback_status ) :
+ import pandokia.text_table as tt
+
+ c = db.cursor()
+ c.execute("select max(depth) as d, tablename from status where run = ? group by tablename order by d asc",(run_name,))
+ table_list = [ x for x in c ]
+ # table_list contains ( depth, tablename )
+
+ print """
+ -- N = not started
+ -- S = started, not finished
+ -- P = prereq not satisfied, so not attempted
+ -- 0-255 = exit code
+"""
+
+ for depth, tablename in table_list :
+ print "------"
+ print tablename
+ t = tt.text_table()
+
+ row = 0
+ t.define_column('-') # the command name in column 0
+
+ c.execute("select distinct host from status where tablename = ? and run = ? order by host asc",(tablename, run_name))
+ for host, in c :
+ t.define_column(host)
+ t.set_value(row, host, host)
+
+ c.execute("""select cmd, host, depth, status, start_time, end_time, notes from status
+ where tablename = ? and run = ? order by depth, cmd asc
+ """, ( tablename, run_name ) )
+
+ prev_cmd = None
+ for x in c :
+ cmd, host, depth, status, start_time, end_time, notes = x
+ if cmd != prev_cmd :
+ row = row + 1
+ t.set_value(row, 0, cmd)
+ prev_cmd = cmd
+ t.set_value( row, host, info_callback( tablename, cmd, host, status ) )
+
+ t.pad()
+
+ print t.get_trac_wiki()
+
+#####
+
+if __name__ == '__main__' :
+ main()
diff --git a/steuermann/specfile.exy b/steuermann/specfile.exy
new file mode 100644
index 0000000..8dd42a3
--- /dev/null
+++ b/steuermann/specfile.exy
@@ -0,0 +1,68 @@
+
+# Parse a Steuermann Spec file
+#
+# This is an exyapps grammer in specfile.exy
+
+%%
+parser specfile:
+ ignore: "[ \r\t\n]+"
+ ignore: "#.*\n"
+
+ token END: "$"
+ token TABLE: "TABLE"
+ token HOST : "HOST"
+ token CMD: "CMD"
+ token OPT: "OPT"
+ token AFTER: "AFTER"
+ token name: "[a-zA-Z0-9_.-]+"
+ token STAR: "\*"
+ token cmdname: "[a-zA-Z0-9_.-]+"
+ token tablename: "[a-zA-Z0-9_.-]+"
+ token wildname: "[*?a-zA-Z0-9_.-]+"
+ token RUN: "RUN"
+ token string: '"[^"]*"'
+ token SLASH: "/"
+ token COLON: ":"
+
+ rule start: table_list END {{ return table_list }}
+
+ rule table_list: table_section [ table_list ]
+
+ rule table_section:
+ TABLE tablename {{ table_name = tablename }}
+ HOST
+ {{ hostlist = [ ] }}
+ ( name {{ hostlist.append(name) }} )+
+ command_list
+ # command_list is a list of (command, pos) where command is the text from the file and pos is the location in the file
+ {{ self.data.add_command_list( table_name, hostlist, command_list ) }}
+
+ rule command_list:
+ # one or more commands, appended together into a list
+ {{ cmlist = [ ] }}
+ command {{ cmlist.append( command ) }}
+ [ command_list {{ cmlist += command_list }} ]
+ {{ return cmlist }}
+
+ | {{ return [ ] }}
+
+ rule command:
+ # a single command, including any number of AFTER clauses
+ CMD {{ cmd_pos = self._scanner.get_pos() }} cmdname {{ cmd_name=cmdname; script=cmdname; x_after_clause = [ ] }}
+ [ RUN string {{ script = string[1:-1] }} ]
+ ( {{ after_pos = self._scanner.get_pos() }} AFTER optword after_spec {{ x_after_clause.append( (after_spec,optword, after_pos) ) }} )*
+ {{ return ( cmd_name, script, x_after_clause, cmd_pos ) }}
+
+ # in the AFTER clause, you can say OPT to mean the node is optional (not an error if it does not exist)
+ rule optword:
+ OPT {{ return 0 }}
+ | {{ return 1 }}
+
+ rule after_spec:
+ wildname {{ rval = wildname }}
+ [ COLON wildname {{ rval = rval + ':' + wildname }} ]
+ [ SLASH wildname {{ rval = rval + '/' + wildname }} ]
+ {{ return rval }}
+
+%%
+
diff --git a/tests/misc.py b/tests/misc.py
new file mode 100644
index 0000000..c1d12aa
--- /dev/null
+++ b/tests/misc.py
@@ -0,0 +1,43 @@
+
+import nodes
+
+yes_list = (
+ ( 'a:b/c', 'a:b/c' ),
+ ( 'a:b/*', 'a:b/c' ),
+ ( 'a:*/c', 'a:b/c' ),
+ ( 'a:*/*', 'a:b/c' ),
+ ( '*:b/c', 'a:b/c' ),
+ ( '*:b/*', 'a:b/c' ),
+ ( '*:*/c', 'a:b/c' ),
+ ( '*:*/*', 'a:b/c' ),
+
+ ( '[a-z]:*/*', 'a:b/c' ),
+
+ ( 'a?:*/*', 'ax:b/c' ),
+ ( 'a?:*/*', 'ay:b/c' ),
+
+ ( '*:*/*', 'xasdadsf:agasdgg/asdgasdg' ),
+ )
+
+no_list = (
+ ( '[A-Z]:*/*', 'a:b/c' ),
+ ( 'a?:*/*', 'a:b/c' ),
+ ( 'a:b/*', 'a:x/y' ),
+ ( '*:c/*', 'a:b/y' ),
+ ( '*:b/y', 'a:b/c' ),
+ ( 'a:b/y', 'a:b/c' ),
+ )
+
+def test_wildcard_name() :
+ def yes( a, b ) :
+ assert nodes.wildcard_name( a, b )
+
+ def no( a, b ) :
+ assert not nodes.wildcard_name( a, b )
+
+ for x in yes_list :
+ yield yes, x[0], x[1]
+
+ for x in no_list :
+ yield no, x[0], x[1]
+
generated by cgit (git 2.34.1) at 2025-11-04 07:34:26 -0500