diff options
| -rw-r--r-- | daily.in | 76 | ||||
| -rw-r--r-- | daily.sr | 17 | ||||
| -rw-r--r-- | scripts/steuermann_report.cgi | 91 | ||||
| -rw-r--r-- | setup.py | 36 | ||||
| -rw-r--r-- | steuermann/config.py | 5 | ||||
| -rw-r--r-- | steuermann/hosts.ini | 44 | ||||
| -rw-r--r-- | steuermann/report.py | 81 | ||||
| -rw-r--r-- | steuermann/run.py | 21 | ||||
| -rw-r--r-- | steuermann/run_all.py | 12 |
9 files changed, 262 insertions, 121 deletions
diff --git a/daily.in b/daily.in deleted file mode 100644 index ae0c843..0000000 --- a/daily.in +++ /dev/null @@ -1,76 +0,0 @@ -# one approach we might use for ssb builds - -TABLE assemble HOST thor - CMD svnsync RUN "svnsync" - AFTER OPT assemble/irafx_update - CMD stsci_python RUN "assemble_stsci_python" - AFTER svnsync - CMD stsci_iraf RUN "assemble_stsci_iraf" - AFTER svnsync - CMD astrolib RUN "assemble_astrolib" - AFTER svnsync - CMD axe RUN "assemble_axe" - AFTER svnsync - CMD hstcal RUN "assemble_hstcal" - AFTER svnsync - -# all machines -TABLE build HOST rhe4-32 rhe4-64 rhe5-64 leopard snow_leopard - - CMD python2.7 RUN "build_stsci_python dev2.7" - AFTER *:assemble/stsci_python - - CMD hstcal RUN "build_hstcal" - AFTER *:assemble/hstcal - -TABLE build HOST rhe-64 - - CMD python2.6 RUN "build_stsci_python dev2.6" - AFTER *:assemble/stsci_python - - CMD python2.5 RUN "build_stsci_python dev2.5" - AFTER *:assemble/stsci_python - - -# 32 bit -TABLE build32 HOST rhe4-32 leopard - - CMD stsci_iraf RUN "build_stsci_iraf" - AFTER *:assemble/stsci_iraf - - CMD axe RUN "build_axe" - AFTER *:assemble/axe - AFTER build32/stsci_iraf - -# copy 32 bit exe to 64 bit machine -TABLE build64 HOST rhe4-64 - CMD iraf32_hack RUN "iraf32hack rhe4-32" - -TABLE build64 HOST rhe5-64 - CMD iraf32_hack RUN "iraf32hack rhe4-32" - -TABLE build64 HOST snow_leopard - CMD iraf32_hack RUN "iraf32hack leopard" - - -# -TABLE build_finished HOST rhe4-32 leopard - CMD finished2.7 RUN "echo done" - AFTER build/python2.7 - AFTER build/hstcal - AFTER build32/stsci_iraf - AFTER build32/axe - -TABLE build_finished HOST rhe4-64 rhe5-64 snow_leopard - CMD finished2.7 RUN "echo done" - AFTER build/python2.7 - AFTER build/hstcal - AFTER build64/iraf32_hack - - -# -TABLE web_updates HOST thor - CMD pyraf RUN "exit 1" - AFTER *:assemble/svnsync - CMD pyfits RUN "exit 1" - AFTER *:assemble/svnsync @@ -1,7 +1,9 @@ +# one approach we might use for ssb builds + TABLE assemble HOST thor - CMD svnsync RUN "svnsync" + CMD svnsync RUN "/eng/ssb/auto/prog/assemble_svnsync" AFTER OPT assemble/irafx_update - CMD stsci_python RUN "assemble_stsci_python" + CMD stsci_python RUN "/eng/ssb/auto/prog/assemble_stsci_python" AFTER svnsync CMD stsci_iraf RUN "assemble_stsci_iraf" AFTER svnsync @@ -13,7 +15,7 @@ TABLE assemble HOST thor AFTER svnsync # all machines -TABLE build HOST rhe4-32 rhe4-64 rhe5-64 leopard snow_leopard +TABLE build HOST rhe4-32 rhe4-64 rhe5-64 leopard snow-leopard CMD python2.7 RUN "build_stsci_python dev2.7" AFTER *:assemble/stsci_python @@ -21,7 +23,7 @@ TABLE build HOST rhe4-32 rhe4-64 rhe5-64 leopard snow_leopard CMD hstcal RUN "build_hstcal" AFTER *:assemble/hstcal -TABLE build HOST rhe-64 +TABLE build HOST rhe5-64 CMD python2.6 RUN "build_stsci_python dev2.6" AFTER *:assemble/stsci_python @@ -43,12 +45,15 @@ TABLE build32 HOST rhe4-32 leopard # copy 32 bit exe to 64 bit machine TABLE build64 HOST rhe4-64 CMD iraf32_hack RUN "iraf32hack rhe4-32" + AFTER rhe4-32:build32/stsci_iraf TABLE build64 HOST rhe5-64 CMD iraf32_hack RUN "iraf32hack rhe4-32" + AFTER rhe4-32:build32/stsci_iraf -TABLE build64 HOST snow_leopard +TABLE build64 HOST snow-leopard CMD iraf32_hack RUN "iraf32hack leopard" + AFTER leopard:build32/stsci_iraf # @@ -59,7 +64,7 @@ TABLE build_finished HOST rhe4-32 leopard AFTER build32/stsci_iraf AFTER build32/axe -TABLE build_finished HOST rhe4-64 rhe5-64 snow_leopard +TABLE build_finished HOST rhe4-64 rhe5-64 snow-leopard CMD finished2.7 RUN "echo done" AFTER build/python2.7 AFTER build/hstcal diff --git a/scripts/steuermann_report.cgi b/scripts/steuermann_report.cgi index fb82b89..a25d407 100644 --- a/scripts/steuermann_report.cgi +++ b/scripts/steuermann_report.cgi @@ -2,33 +2,108 @@ import cgi import cgitb +import os +import sys +import re +import datetime + + +STEUERMANN_DIR_HERE +sys.path.insert(0, addpath) + +import steuermann.config cgitb.enable() form = cgi.FieldStorage(keep_blank_values=1) cginame = os.getenv("SCRIPT_NAME") -import sqlite3 +def sqltime(arg) : + if arg is None : + return None + + if '.' in arg : + x = arg.split('.') + d = datetime.datetime.strptime(x[0],'%Y-%m-%d %H:%M:%S') + d = d.replace(microsecond=int((x[1]+'000000')[0:6])) + else : + x = time.strptime(arg,'%Y-%m-%d %H:%M:%S') + d = datetime.datetime(year=x[0], month=x[1], day=x[2], + hour=x[3], minute=x[4], second=x[5] ) + # not in 2.4: + # d = datetime.datetime.strptime(arg,'%Y-%m-%d %H:%M:%S') + return d + if not 'action' in form : print 'content-type: text/html' print '' - db = sqlite3.connect('db.sr') + db = steuermann.config.open_db() c = db.cursor() - c.execute('SELECT DISTINCT run FROM status ORDER BY run ASC') + c.execute('SELECT DISTINCT run FROM status ORDER BY run DESC') for run, in c : - print "<a href=%s?action=report&run=%s>%s</a><br>"%(cginame, run, run) - return + print "<a href=%s?action=status&run=%s>%s</a><br>"%(cginame, run, run) + sys.exit(0) action = form['action'].value -if action == 'report' : +if action == 'status' : + db = steuermann.config.open_db() import steuermann.report + steuermann.report.cginame = cginame print 'content-type: text/html' print '' run = form['run'].value - print steuermann.report.report_html( db, run ) - return + print steuermann.report.report_html( db, run, info_callback=steuermann.report.info_callback_gui ) + sys.exit(0) + +elif action == 'log' : + print 'content-type: text/plain' + print '' + + # crack apart the parameter run/host:table/cmd + name = re.match('(.*)/(.*):(.*)/(.*)', form['name'].value) + run = name.group(1) + host = name.group(2) + table = name.group(3) + cmd = name.group(4) + + + db = steuermann.config.open_db() + c = db.cursor() + c.execute("SELECT status, start_time, end_time, notes FROM status WHERE run = ? AND host = ? AND tablename = ? AND cmd = ?",( + run, host, table, cmd ) ) + x = c.fetchone() + if x is None : + print "No such record in database",run,host,table,cmd + sys.exit(0) + + status, start_time, end_time, notes = x + + print "%s %s:%s/%s"%(run, host, table, cmd) + print "status: %s"%status + print "" + print "start: %s"%start_time + print "end : %s"%end_time + start_time = sqltime(start_time) + end_time = sqltime(end_time) + if isinstance(end_time,datetime.datetime) and isinstance(end_time,datetime.datetime) : + print "dur : %s"%(end_time-start_time) + + if not notes is None : + print "notes:" + for x in [ ' ' + x for x in notes.split('\n') ] : + print x + print "" + print "--------------------" + filename = '%s/%s/%s:%s.%s.log'%(steuermann.config.logdir,run,host,table,cmd) + f=open(filename,'r') + while 1 : + x = f.read(65536) + if x == '' : + break + sys.stdout.write(x) + sys.exit(0) print 'content-type: text/html' print '' @@ -9,11 +9,15 @@ f.close() print version +command_list = [ 'smc', 'steuermann_report.cgi' ] +use_usr_bin_env = [ ] +dir_set = 'addpath = "%s"\n' + args = { 'version' : "2.0dev", 'description' : "Steuermann Continuous Integration Control System", 'author' : "Mark Sienkiewicz", - 'scripts' : ['scripts/smc'], + 'scripts' : ['scripts/smc', 'scripts/steuermann_report.cgi'], 'package_dir' : { 'steuermann' : 'steuermann', }, 'url' : 'https://svn.stsci.edu/trac/ssb/etal/wiki/Steuermann', 'license': 'BSD', @@ -25,3 +29,33 @@ d = distutils.core.setup( **args ) + +def fix_script(name) : + fname = script_dir + "/" + name + + f=open(fname,"r") + l = f.readlines() + if name in use_usr_bin_env : + l[0] = '#!/usr/bin/env python\n' + for count, line in enumerate(l) : + if line.startswith("STEUERMANN_DIR_HERE") : + l[count] = dir_set % lib_dir + f.close() + + f=open(fname,"w") + f.writelines(l) + f.close() + +if 'install' in d.command_obj : + # they did an install + script_dir = d.command_obj['install'].install_scripts + lib_dir = d.command_obj['install'].install_lib + print 'scripts went to', script_dir + print 'python went to', lib_dir + for x in command_list : + fix_script(x) + print 'set path = ( %s $path )' % script_dir + print 'setenv PYTHONPATH %s:$PYTHONPATH' % lib_dir +else : + print "no install" + diff --git a/steuermann/config.py b/steuermann/config.py new file mode 100644 index 0000000..c422dc2 --- /dev/null +++ b/steuermann/config.py @@ -0,0 +1,5 @@ +def open_db() : + import sqlite3 + return sqlite3.connect('/ssbwebv1/data2/steuermann/steuermann.sb') + +logdir = '/ssbwebv1/data2/steuermann/logs' diff --git a/steuermann/hosts.ini b/steuermann/hosts.ini index 61231df..c64e2aa 100644 --- a/steuermann/hosts.ini +++ b/steuermann/hosts.ini @@ -12,21 +12,41 @@ ; %(cmd)s - ; %(foo)s - if you include a foo= line ; -[arzach] -workdir=/arzach/data1/sienkiew +[linux:csh] +hostname=no_such_machine +run=[ 'ssh', '-q', '%(hostname)s', 'source .steuermann.%(hostname)s; cd %(workdir)s; hostname; %(script)s ' ] +workdir=/tmp -[thor] -workdir=/thor/data2/sienkiew -[herbert] -workdir=/herbert/data1/sienkiew +[rhe4-32] +hostname=herbert +like=linux:csh -[jwcalibdev] -workdir=/data1/sienkiew/ur_work +[rhe4-64] +hostname=thor +like=linux:csh + +[rhe5-64] +hostname=arzach +like=linux:csh + +[leopard] +hostname=bond +like=linux:csh + +[snow-leopard] +hostname=cadeau +like=linux:csh + + +[arzach] +hostname=arzach +like=linux:csh + +[thor] +hostname=thor +like=linux:csh -; There is a section [ALL] that is used with every machine name -[ALL] -run=[ 'ssh', '-q', '%(host)s', 'cd %(workdir)s; %(script)s' ] [host_a] run=[ 'sleep', '3' ] @@ -41,3 +61,5 @@ run=[ 'sleep', '3' ] run=[ 'sleep', '3' ] +; There is a section [ALL] that is used with every machine name +[ALL] diff --git a/steuermann/report.py b/steuermann/report.py index e1df5ce..e1d2971 100644 --- a/steuermann/report.py +++ b/steuermann/report.py @@ -5,10 +5,58 @@ Generate reports from the database import time import sys -import sqlite3 import pandokia.text_table as text_table import StringIO +# this will be reset by the cgi main program if we are in a real cgi +cginame = 'arf.cgi' + +# + +def info_callback_status( db, run, tablename, host, cmd ) : + c = db.cursor() + c.execute("SELECT status FROM status WHERE run = ? AND host = ? AND tablename = ? AND cmd = ?",( + run, host, tablename, cmd ) ) + status, = c.fetchone() + return status + +# + +def info_callback_gui( db, run, tablename, host, cmd ) : + c = db.cursor() + c.execute("SELECT status, start_time, end_time FROM status WHERE run = ? AND host = ? AND tablename = ? AND cmd = ?",( + run, host, tablename, cmd ) ) + x = c.fetchone() + if x is None : + return '' + status, start_time, end_time = x + if start_time is None : + start_time = '' + if end_time is None : + end_time = '' + + # t_result = '%s %s %s'%(status, start_time, end_time ) + t_result = status + + if status != '0' and status != 'N' : + status = '<font color=red>%s</font>'%status + + if status != 'N' and status != 'P' : + link = "<a href='%s?action=log&name=%s/%s:%s/%s'>*</a>"%(cginame, run, host, tablename, cmd ) + else : + link = '' + + # h_result = '%s %s %s %s'%(link, status, start_time, end_time) + h_result = '%s %s'%(link, status ) + + return ( t_result, h_result ) + +# + +def info_callback_debug_table_cell( db, run, tablename, cmd, host ) : + return '%s %s %s %s' % ( run, host, tablename, cmd ) + +# def get_table_list( db, run_name ) : c = db.cursor() @@ -17,39 +65,46 @@ def get_table_list( db, run_name ) : # table_list contains ( depth, tablename ) return table_list -def get_table( db, run_name, tablename, info_callback ) : +def get_table( db, run_name, tablename, info_callback, showdepth=0 ) : t = text_table.text_table() + t.set_html_table_attributes('border=1') - row = 0 - t.define_column('-') # the command name in column 0 + t.define_column('-',html=' ',showname='-') # the command name in column 0 + + if showdepth : + t.define_column('depth') c = db.cursor() c.execute("select distinct host from status where tablename = ? and run = ? order by host asc",(tablename, run_name)) for host, in c : t.define_column(host) - t.set_value(row, host, host) - c.execute("""select cmd, host, depth, status, start_time, end_time, notes from status where tablename = ? and run = ? order by depth, cmd asc """, ( tablename, run_name ) ) + row = -1 prev_cmd = None for x in c : cmd, host, depth, status, start_time, end_time, notes = x if cmd != prev_cmd : row = row + 1 t.set_value(row, 0, cmd) + t.set_value(row, 'depth', depth) prev_cmd = cmd - t.set_value( row, host, info_callback( tablename, cmd, host, status ) ) + + info = info_callback( db, run_name, tablename, host, cmd ) + if isinstance(info, tuple) : + t.set_value( row, host, text=info[0], html=info[1] ) + else : + t.set_value( row, host, info) t.pad() return t -def info_callback_status( tablename, cmd, host, status ) : - return status +# def report_text( db, run_name, info_callback = info_callback_status ) : @@ -68,6 +123,8 @@ def report_text( db, run_name, info_callback = info_callback_status ) : return s.getvalue() +# + def report_html( db, run_name, info_callback = info_callback_status, hlevel=1 ) : s = StringIO.StringIO() s.write('<h%d>%s</h%d>\n'%(hlevel,run_name,hlevel)) @@ -78,13 +135,15 @@ def report_html( db, run_name, info_callback = info_callback_status, hlevel=1 ) for depth, tablename in table_list : s.write('<h%d>%s</h%d>\n'%(hlevel,tablename,hlevel)) - t = get_table( db, run_name, tablename, info_callback ) + t = get_table( db, run_name, tablename, info_callback, showdepth=1 ) s.write(t.get_html()) return s.getvalue() +# def main() : - db = sqlite3.connect('sr.db') + import steuermann.config + db = steuermann.config.open_db() print report_html( db, 'arf2011-08-30 16:52:23.928381' ) if __name__ == '__main__' : diff --git a/steuermann/run.py b/steuermann/run.py index 6526e1a..f3e86b3 100644 --- a/steuermann/run.py +++ b/steuermann/run.py @@ -32,14 +32,15 @@ class runner(object): host_info = { } # dir where we write our logs - logdir = 'logs' + logdir = '' ##### # - def __init__( self, nodes ) : + def __init__( self, nodes, logdir ) : self.node_index = nodes self.load_host_info() + self.logdir = logdir ##### # start a process @@ -85,7 +86,7 @@ class runner(object): pass # create a name for the log file, but do not use / in the name - logfile_name = "%s/%s.log"%( logdir, node.name.replace('/','+') ) + logfile_name = "%s/%s.log"%( logdir, node.name.replace('/','.') ) # open the log file, write initial notes logfile=open(logfile_name,"w") @@ -197,8 +198,22 @@ class runner(object): if x == 'ALL' : continue + # start with a dict that contains what is in ALL d = all_dict.copy() + + # get what there is to know about host x self._host_get_names(cfg, x, d) + + # if it is like some other host, start over using ALL, then + # the LIKE host, then our own information + if 'like' in d : + like = d['like'] + d = all_dict.copy() + self._host_get_names(cfg, like, d) + self._host_get_names(cfg, x, d) + del d['like'] + + print x,d self.host_info[x] = d del cfg diff --git a/steuermann/run_all.py b/steuermann/run_all.py index 59797cb..d94cafa 100644 --- a/steuermann/run_all.py +++ b/steuermann/run_all.py @@ -5,7 +5,6 @@ run everything in a set of command files import time import sys -import sqlite3 import os.path import datetime @@ -13,6 +12,8 @@ import run import report import nodes +import steuermann.config + try : import readline @@ -39,8 +40,8 @@ def main() : di_nodes = nodes.read_file_list( sys.argv[2:] ) xnodes = di_nodes.node_index - run_name = 'arf' + str(datetime.datetime.now()) - db = sqlite3.connect('sr.db') + run_name = str(datetime.datetime.now()).replace(' ','_') + db = steuermann.config.open_db() register_database(db, run_name, xnodes) n = sys.argv[1] @@ -113,7 +114,7 @@ pre node show what must come before a node def run_interactive( xnodes, run_name, db) : - runner = run.runner( xnodes ) + runner = run.runner( xnodes, steuermann.config.logdir ) for x in xnodes : xnodes[x].finished = 0 @@ -329,6 +330,7 @@ def run_step( runner, xnodes, run_name, db ) : if not who_exited : break + print "SOMETHING EXITED",who_exited # yes, something exited - no sleep, and keep running no_sleep = 1 keep_running = 1 @@ -340,7 +342,7 @@ def run_step( runner, xnodes, run_name, db ) : db.execute("UPDATE status SET end_time = ?, status = ? WHERE ( run = ? AND host = ? AND tablename = ? AND cmd = ? )", ( str(datetime.datetime.now()), who_exited[1], run_name, x_host, x_table, x_cmd ) ) - + db.commit() runner.display_procs() |