diff options
| -rw-r--r-- | Makefile | 1 | ||||
| -rw-r--r-- | daily.sm | 79 | ||||
| -rw-r--r-- | dev.sm | 61 | ||||
| -rw-r--r-- | init.sm | 12 | ||||
| -rw-r--r-- | steuermann/config.py | 2 | ||||
| -rw-r--r-- | steuermann/db.sql | 3 | ||||
| -rw-r--r-- | steuermann/hosts.ini | 21 | ||||
| -rw-r--r-- | steuermann/nodes.py | 10 | ||||
| -rw-r--r-- | steuermann/report.py | 37 | ||||
| -rw-r--r-- | steuermann/run.py | 3 | ||||
| -rw-r--r-- | steuermann/run_all.py | 75 | ||||
| -rw-r--r-- | steuermann/specfile.exy | 27 | ||||
| -rw-r--r-- | tests/hosts.py | 10 |
13 files changed, 202 insertions, 139 deletions
@@ -9,4 +9,5 @@ sr.db: steuermann/db.sql clean: rm -f sr.db + rm -f steuermann/specfile.py diff --git a/daily.sm b/daily.sm deleted file mode 100644 index eb0c691..0000000 --- a/daily.sm +++ /dev/null @@ -1,79 +0,0 @@ -# one approach we might use for ssb builds - -TABLE assemble HOST thor - CMD svnsync RUN "/eng/ssb/auto/prog/assemble_svnsync" - AFTER OPT assemble/irafx_update - CMD stsci_python RUN "/eng/ssb/auto/prog/assemble_stsci_python" - AFTER svnsync - CMD stsci_iraf RUN "assemble_stsci_iraf" - AFTER svnsync - CMD astrolib RUN "assemble_astrolib" - AFTER svnsync - CMD axe RUN "assemble_axe" - AFTER svnsync - CMD hstcal RUN "assemble_hstcal" - AFTER svnsync - -# all machines -TABLE build HOST rhe4-32 rhe4-64 rhe5-64 leopard snow-leopard - - CMD python2.7 RUN "build_stsci_python dev2.7" - AFTER *:assemble/stsci_python - - CMD hstcal RUN "build_hstcal" - AFTER *:assemble/hstcal - -TABLE build HOST rhe5-64 - - CMD python2.6 RUN "build_stsci_python dev2.6" - AFTER *:assemble/stsci_python - - CMD python2.5 RUN "build_stsci_python dev2.5" - AFTER *:assemble/stsci_python - - -# 32 bit -TABLE build32 HOST rhe4-32 leopard - - CMD stsci_iraf RUN "build_stsci_iraf" - AFTER *:assemble/stsci_iraf - - CMD axe RUN "build_axe" - AFTER *:assemble/axe - AFTER build32/stsci_iraf - -# copy 32 bit exe to 64 bit machine -TABLE build64 HOST rhe4-64 - CMD iraf32_hack RUN "iraf32hack rhe4-32" - AFTER rhe4-32:build32/stsci_iraf - -TABLE build64 HOST rhe5-64 - CMD iraf32_hack RUN "iraf32hack rhe4-32" - AFTER rhe4-32:build32/stsci_iraf - -TABLE build64 HOST snow-leopard - CMD iraf32_hack RUN "iraf32hack leopard" - AFTER leopard:build32/stsci_iraf - - -# -TABLE build_finished HOST rhe4-32 leopard - CMD finished2.7 RUN "echo done" - AFTER build/python2.7 - AFTER build/hstcal - AFTER build32/stsci_iraf - AFTER build32/axe - -TABLE build_finished HOST rhe4-64 rhe5-64 snow-leopard - CMD finished2.7 RUN "echo done" - AFTER build/python2.7 - AFTER build/hstcal - AFTER build64/iraf32_hack - - -# -TABLE web_updates HOST thor - CMD pyraf RUN "exit 1" - AFTER *:assemble/svnsync - CMD pyfits RUN "exit 1" - AFTER *:assemble/svnsync @@ -0,0 +1,61 @@ + +TABLE assemble HOST rhe5-64 + CMD dev.stsci_python RUN "assemble_stsci_python dev" + AFTER init/* + + CMD dev.stsci_iraf RUN "assemble_stsci_iraf dev" + AFTER init/* + + CMD dev.axe RUN "assemble_axe dev" + AFTER init/* + + CMD dev.hstcal RUN "assemble_hstcal dev" + AFTER init/* + +TABLE build HOST rhe4-32 rhe4-64 rhe5-64 leopard snow-leopard + CMD dev.py2.7 RUN "build_stsci_python dev 2.7" + AFTER init/* + AFTER *:assemble/dev.stsci_python + CMD dev.hstcal RUN "build_hstcal dev" + AFTER init/* + AFTER *:assemble/dev.hstcal + +TABLE build HOST rhe5-64 + CMD dev.py2.6 RUN "build_stsci_python dev 2.6" + AFTER init/* + AFTER *:assemble/dev.stsci_python + + CMD dev.py2.5 RUN "build_stsci_python dev 2.5" + AFTER init/* + AFTER *:assemble/dev.stsci_python + +TABLE build HOST rhe4-32 leopard + CMD dev.axe RUN "build_axe dev" + AFTER init/* + AFTER *:assemble/dev.axe + + CMD dev.stsci_iraf RUN "build_stsci_iraf dev" + AFTER init/* + AFTER *:assemble/dev.stsci_iraf + AFTER build/dev.axe + + CMD dev.stsci_iraf_log RUN "build_stsci_iraf_log dev" + AFTER init/* + AFTER build/dev.stsci_iraf + + CMD dev.stsci_iraf_help RUN "build_stsci_iraf_help dev" + AFTER init/* + AFTER build/dev.stsci_iraf + + +TABLE build HOST rhe4-64 rhe5-64 + CMD dev.stsci_iraf_64hack RUN "build_stsci_iraf_64hack dev herbert" + AFTER rhe4-32:build/dev.stsci_iraf* + +TABLE build HOST snow-leopard + CMD dev.stsci_iraf_64hack RUN "build_stsci_iraf_64hack dev cadeau" + AFTER rhe4-32:build/dev.stsci_iraf* + +TABLE build HOST rhe5-64 + CMD dev.stsci_python_sphinxdocs RUN "build_sphinxdocs dev 2.7" + AFTER build/dev.py2.7 @@ -0,0 +1,12 @@ +TABLE init HOST leopard snow-leopard rhe5-64 rhe4-64 rhe4-32 + CMD sendscripts LOCAL "/eng/ssb/auto/steuermann_scripts/init_sendscripts %(hostname)s %(workdir)s" + CMD sysstat RUN "sysstat" + AFTER sendscripts + +TABLE init HOST rhe5-64 + CMD svnsync RUN "assemble_svnsync" + AFTER OPT init/irafx_update + AFTER sendscripts + AFTER sysstat + + diff --git a/steuermann/config.py b/steuermann/config.py index c422dc2..7419703 100644 --- a/steuermann/config.py +++ b/steuermann/config.py @@ -1,5 +1,5 @@ def open_db() : import sqlite3 - return sqlite3.connect('/ssbwebv1/data2/steuermann/steuermann.sb') + return sqlite3.connect('/ssbwebv1/data2/steuermann/steuermann.db') logdir = '/ssbwebv1/data2/steuermann/logs' diff --git a/steuermann/db.sql b/steuermann/db.sql index 664f5df..dd6a569 100644 --- a/steuermann/db.sql +++ b/steuermann/db.sql @@ -16,7 +16,8 @@ CREATE TABLE status ( status VARCHAR(5), -- N = not started - -- S = started, not finished + -- R = started, not finished + -- S = skipped -- P = prereq not satisfied, so not attempted -- 0-255 = exit code diff --git a/steuermann/hosts.ini b/steuermann/hosts.ini index d0c5729..0f0c043 100644 --- a/steuermann/hosts.ini +++ b/steuermann/hosts.ini @@ -16,19 +16,22 @@ ; definitions common to various operating system environments [all] +hostname=no_such_machine local=[ 'sh', '-c', '%(script)s' ] maxproc=2 [linux:csh] -hostname=no_such_machine -run=[ 'ssh', '-q', '%(hostname)s', 'source .steuermann.%(hostname)s; cd %(workdir)s; hostname; %(script)s ' ] +; for CSH +; +; -q = quiet +; -x = do not forward X windows; prevents errors locking .Xauthority when lots of ssh come in at the same time +run=[ 'ssh', '-q', '-x', '%(hostname)s', ' cd %(workdir)s; set node=%(node)s; source bin/.steuermann.%(hostname)s; %(script)s; show_status $status ' ] like=all [mac:csh] like=linux:csh [windows:cmd] -hostname=no_such_machine run=[ 'python', '-m', 'steuermann.windows_comm', '%(hostname)s', '%(script)s' ] like=all @@ -56,37 +59,37 @@ like=cadeau [herbert] hostname=herbert like=linux:csh -workdir=/herbert/data1/sienkiew/steuermann +workdir=/herbert/data1/iraf/steuermann maxproc=4 [thor] hostname=thor like=linux:csh -workdir=/thor/data2/sienkiew/steuermann +workdir=/thor/data2/iraf/steuermann maxproc=4 [arzach] hostname=arzach like=linux:csh -workdir=/arzach/data1/sienkiew/steuermann +workdir=/arzach/data1/iraf/steuermann maxproc=4 [bond] hostname=bond like=mac:csh -workdir=/Users/sienkiew/work/steuermann +workdir=/Users/iraf/work/steuermann maxproc=8 [cadeau] hostname=cadeau like=mac:csh -workdir=/Users/sienkiew/work/steuermann +workdir=/Users/iraf/work/steuermann maxproc=8 [banana] hostname=banana like=mac:csh -workdir=/Users/sienkiew/work/steuermann +workdir=/Users/iraf/work/steuermann maxproc=4 ; There is a section [ALL] that is used with every machine name diff --git a/steuermann/nodes.py b/steuermann/nodes.py index 0173793..5e9852a 100644 --- a/steuermann/nodes.py +++ b/steuermann/nodes.py @@ -3,7 +3,7 @@ Stuff related to the tree structure of the command set ''' import fnmatch -import exyapps.runtime +#import exyapps.runtime ##### @@ -55,15 +55,11 @@ class command_tree(object): # make the actual connection between nodes def connect( self, before, after, required, line ) : - if not after in self.node_index : + if not before in self.node_index : if required : print "error: %s happens after non-existant %s - line %s"%(before,after,line) return - if not before in self.node_index : - print "error: before node %s does not exist %s"%(before,line) - return - if not after in self.node_index : print "error: after node %s does not exist %s"%(after,line) return @@ -309,7 +305,7 @@ def read_file_list( file_list ) : current_file_name = x sc = specfile.specfileScanner( open(x,'r').read() ) p = specfile.specfile( scanner=sc, data=di ) - result = exyapps.runtime.wrap_error_reporter( p, 'start' ) + result = specfile.wrap_error_reporter( p, 'start' ) di.finish() return di diff --git a/steuermann/report.py b/steuermann/report.py index ca8d064..82da984 100644 --- a/steuermann/report.py +++ b/steuermann/report.py @@ -6,6 +6,7 @@ Generate reports from the database import time import sys import pandokia.text_table as text_table +import pandokia.common import StringIO # this will be reset by the cgi main program if we are in a real cgi @@ -22,6 +23,10 @@ def info_callback_status( db, run, tablename, host, cmd ) : # +ok_status = ( '0', 'N', 'S' ) + +simple_status = ( 'N', 'P', 'S', 'W' ) + def info_callback_gui( db, run, tablename, host, cmd ) : c = db.cursor() c.execute("SELECT status, start_time, end_time FROM status WHERE run = ? AND host = ? AND tablename = ? AND cmd = ?",( @@ -38,16 +43,32 @@ def info_callback_gui( db, run, tablename, host, cmd ) : # t_result = '%s %s %s'%(status, start_time, end_time ) t_result = status - if status != '0' and status != 'N' : - status = '<font color=red>%s</font>'%status - - if status != 'N' and status != 'P' : - link = "<a href='%s?action=log&name=%s/%s:%s/%s'>*</a>"%(cginame, run, host, tablename, cmd ) + if status not in ok_status : + d_status = '<font color=red>%s</font>'%status else : - link = '' + d_status = status - # h_result = '%s %s %s %s'%(link, status, start_time, end_time) - h_result = '%s %s'%(link, status ) + if status in simple_status : + link = '%s'%d_status + else : + link = " <a href='%s?action=log&name=%s/%s:%s/%s'>%s</a>"%(cginame, run, host, tablename, cmd, d_status ) + + if not status in simple_status : + try : + st = pandokia.common.parse_time(start_time) + st = '%d:%02d'%(st.hour,st.minute) + except ValueError : + st = '?' + try : + et = pandokia.common.parse_time(end_time ) + et = '%d:%02d'%(et.hour,et.minute) + except ValueError : + et = '?' + + link = link + " : " + st + " - " + et + + # h_result = '%s %s %s %s'%(link, d_status, start_time, end_time) + h_result = link return ( t_result, h_result ) diff --git a/steuermann/run.py b/steuermann/run.py index cb2af58..36cda80 100644 --- a/steuermann/run.py +++ b/steuermann/run.py @@ -86,7 +86,8 @@ class runner(object): script_type=node.script_type, host=node.host, table=node.table, - cmd=node.cmd + cmd=node.cmd, + node=node.name, ) if debug : diff --git a/steuermann/run_all.py b/steuermann/run_all.py index 1c6fe28..e7bff61 100644 --- a/steuermann/run_all.py +++ b/steuermann/run_all.py @@ -14,6 +14,8 @@ import nodes import steuermann.config +import pandokia.helpers.easyargs as easyargs + try : import readline @@ -35,18 +37,20 @@ def main() : import atexit atexit.register(readline.write_history_file, history) + opt, args = easyargs.get( { '-a' : '--all', + '--all' : '-a', + } ) + + # # - all = sys.argv[1] == '-a' - if all : - di_nodes = nodes.read_file_list( sys.argv[2:] ) - else : - di_nodes = nodes.read_file_list( sys.argv[1:] ) + all = '--all' in opt + + di_nodes = nodes.read_file_list( args ) xnodes = di_nodes.node_index run_name = str(datetime.datetime.now()).replace(' ','_') db = steuermann.config.open_db() - register_database(db, run_name, xnodes) if all : run_all(xnodes, run_name, db) @@ -83,11 +87,16 @@ def do_flag( xnodes, name, recursive, fn, verbose ) : raise Exception() def set_want( node ) : + # if we said we want it, mark it as wanted and don't skip node.wanted = 1 node.skip = 0 def set_skip( node ) : - node.wanted = 0 + # If we want to skip it, mark it as IS wanted and skip. + # Wanted makes us try to run it, then skip makes the run a nop. + # This means that stuff that comes after it can run, but only + # at the right point in the sequence. + node.wanted = 1 node.skip = 1 @@ -119,6 +128,8 @@ pre node show what must come before a node def run_interactive( xnodes, run_name, db) : + register_database(db, run_name, xnodes) + runner = run.runner( xnodes, steuermann.config.logdir ) for x in xnodes : @@ -179,6 +190,15 @@ def run_interactive( xnodes, run_name, db) : elif n == 'skip' : cmd_flagging( l, xnodes, set_skip ) + elif n == 'reset' : + print "marking all as not finished" + for x in xnodes : + xnodes[x].finished = 0 + + run_name = str(datetime.datetime.now()).replace(' ','_') + print "new run name",run_name + register_database(db, run_name, xnodes) + elif n == 'list' : print_all = '-a' in l l = sorted ( [ x for x in xnodes ] ) @@ -188,10 +208,17 @@ def run_interactive( xnodes, run_name, db) : if print_all : print " AFTER", ' '.join([ a.name for a in xnodes[x].predecessors ]) - elif n == 'start' : - keep_running = 1 - elif n == 'wait' : + c = db.cursor() + for x in xnodes : + host, tablename, cmd = nodes.crack_name(x) + if xnodes[x].wanted : + status = 'W' + c.execute("UPDATE status SET status = 'W' WHERE run = ? AND host = ? AND tablename = ? AND cmd = ? AND status = 'N'", + (run_name, host, tablename, cmd) ) + + db.commit() + while 1 : ( keep_running, no_sleep ) = run_step( runner, xnodes, run_name, db ) if not keep_running : @@ -274,6 +301,8 @@ def register_database(db, run, xnodes ) : def run_all(xnodes, run_name, db) : + register_database(db, run_name, xnodes) + runner = run.runner( xnodes ) for x in xnodes : @@ -304,12 +333,6 @@ def run_step( runner, xnodes, run_name, db ) : # skip nodes that we do not need to consider running because - # - we explicitly ask to skip it; also mark it finished - # so that things that come after can run - if x.skip : - x.finished = 1 - continue - # - it is not wanted if not x.wanted : continue @@ -337,19 +360,29 @@ def run_step( runner, xnodes, run_name, db ) : if released == len(x.released) : host, table, cmd = nodes.crack_name(x_name) - if runner.run(x, run_name) : - # returns true/false whether it actually ran it - it may not because of resource limits + # we are now ready to let it run. If it is marked skipped, just say it ran really fast. + if x.skip : + x.finished = 1 + no_sleep = 1 + keep_running = 1 db.execute("UPDATE status SET start_time = ?, status = 'S' WHERE ( run = ? AND host = ? AND tablename = ? AND cmd = ? )", - ( str(datetime.datetime.now()), run_name, host, table, cmd ) ) + ( str(datetime.datetime.now()), run_name, host, table, cmd ) ) db.commit() + else : + if runner.run(x, run_name) : + # returns true/false whether it actually ran it - it may not because of resource limits + db.execute("UPDATE status SET start_time = ?, status = 'R' WHERE ( run = ? AND host = ? AND tablename = ? AND cmd = ? )", + ( str(datetime.datetime.now()), run_name, host, table, cmd ) ) + db.commit() + # if anything has exited, we process it and update the status in the database while 1 : who_exited = runner.poll() if not who_exited : break - # print "SOMETHING EXITED",who_exited + print "SOMETHING EXITED",who_exited # yes, something exited - no sleep, and keep running no_sleep = 1 keep_running = 1 @@ -357,8 +390,6 @@ def run_step( runner, xnodes, run_name, db ) : # note who and log it x_host, x_table, x_cmd = nodes.crack_name(who_exited[0]) - xnodes[who_exited[0]].wanted = 0 - db.execute("UPDATE status SET end_time = ?, status = ? WHERE ( run = ? AND host = ? AND tablename = ? AND cmd = ? )", ( str(datetime.datetime.now()), who_exited[1], run_name, x_host, x_table, x_cmd ) ) db.commit() diff --git a/steuermann/specfile.exy b/steuermann/specfile.exy index 5dc715f..7cdf1f3 100644 --- a/steuermann/specfile.exy +++ b/steuermann/specfile.exy @@ -31,10 +31,11 @@ parser specfile: rule table_section: TABLE tablename {{ table_name = tablename }} - HOST - {{ hostlist = [ ] }} - ( name {{ hostlist.append(name) }} )+ - command_list + HOST {{ hostlist = [ ] }} + ( + name {{ hostlist.append(name) }} + )+ + command_list # command_list is a list of (command, pos) where command is the text from the file and pos is the location in the file {{ self.data.add_command_list( table_name, hostlist, command_list ) }} @@ -42,16 +43,22 @@ parser specfile: # one or more commands, appended together into a list {{ cmlist = [ ] }} command {{ cmlist.append( command ) }} - [ command_list {{ cmlist += command_list }} ] + [ + command_list {{ cmlist += command_list }} + ] {{ return cmlist }} - - | {{ return [ ] }} + | {{ return [ ] }} rule command: # a single command, including any number of AFTER clauses - CMD {{ cmd_pos = self._scanner.get_pos() }} cmdname {{ cmd_name=cmdname; script=cmdname; x_after_clause = [ ] }} - [ RUN string {{ script = string[1:-1]; script_type='r' }} | LOCAL string {{ script = string[1:-1]; script_type='l' }} ] - ( {{ after_pos = self._scanner.get_pos() }} AFTER optword after_spec {{ x_after_clause.append( (after_spec,optword, after_pos) ) }} )* + CMD {{ cmd_pos = self._scanner.get_pos() }} + cmdname {{ cmd_name=cmdname; script=cmdname; x_after_clause = [ ] }} + [ RUN string {{ script = string[1:-1]; script_type='r' }} + | LOCAL string {{ script = string[1:-1]; script_type='l' }} + ] + ( {{ after_pos = self._scanner.get_pos() }} + AFTER optword after_spec {{ x_after_clause.append( (after_spec, optword, after_pos) ) }} + )* {{ return ( cmd_name, script, script_type, x_after_clause, cmd_pos ) }} # in the AFTER clause, you can say OPT to mean the node is optional (not an error if it does not exist) diff --git a/tests/hosts.py b/tests/hosts.py index 5fbfa25..c1305a8 100644 --- a/tests/hosts.py +++ b/tests/hosts.py @@ -3,11 +3,19 @@ import steuermann.run as run r = run.runner(0,0) def test_1() : - print r.cfg.sections() + print "SECTIONS",r.cfg.sections() def test_2() : print r.get_host_info('rhe4-32') +def test_3() : + print r.get_host_info('leopard') + +print "XXXXXXXXXX" test_1() +print "XXXXXXXXXX" test_2() +print "XXXXXXXXXX" +test_3() +print "XXXXXXXXXX" |