diff options
| -rw-r--r-- | LICENSE | 28 | ||||
| -rw-r--r-- | MANIFEST.in | 6 | ||||
| -rw-r--r-- | README | 20 | ||||
| -rw-r--r-- | cdbs.sm | 13 | ||||
| -rw-r--r-- | dev.sm | 27 | ||||
| -rwxr-xr-x | go | 46 | ||||
| -rw-r--r-- | init.sm | 8 | ||||
| -rw-r--r-- | scripts/smc | 5 | ||||
| -rw-r--r-- | scripts/smcron | 7 | ||||
| -rw-r--r-- | scripts/steuermann_report.cgi | 186 | ||||
| -rw-r--r-- | setup.py | 50 | ||||
| -rw-r--r-- | steuermann/db.sql | 25 | ||||
| -rw-r--r-- | steuermann/hosts.ini | 27 | ||||
| -rw-r--r-- | steuermann/nodes.py | 4 | ||||
| -rw-r--r-- | steuermann/report.py | 15 | ||||
| -rw-r--r-- | steuermann/run.py | 57 | ||||
| -rw-r--r-- | steuermann/run_all.py | 21 | ||||
| -rw-r--r-- | steuermann/run_cron.py | 82 | ||||
| -rw-r--r-- | x_build.sm | 80 | ||||
| -rw-r--r-- | x_dist.sm | 104 |
20 files changed, 731 insertions, 80 deletions
@@ -0,0 +1,28 @@ +Copyright (C) 2011 Association of Universities for Research in Astronomy (AURA) + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +3. The name of AURA and its representatives may not be used to endorse +or promote products derived from this software without specific prior +written permission. + +THIS SOFTWARE IS PROVIDED BY AURA ``AS IS'' AND ANY EXPRESS OR IMPLIED +WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN +NO EVENT SHALL AURA BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..87d6d04 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,6 @@ +recursive-include docs +recursive-include scripts +recursive-include steuermann +recursive-include tests +include go LICENSE Makefile README + @@ -1,11 +1,23 @@ Steuermann 2.0 -- -requires exyapps, pandokia +To install: -make + requires exyapps, pandokia -python setup.py install + make + # compiles the parser + + python setup.py install + # installs + + Put the installed script steuermann_report.cgi on your web server. + It remembers where the rest was installed. + +To run: + + requires pandokia + + smc [ -a ] [ -r run_name ] file.sm -smc [ -a ] [ -r run_name ] file.sm @@ -0,0 +1,13 @@ +TABLE cdbs HOST bond cadeau + CMD local_copy RUN "cdbs_copy" + AFTER init/* + +TABLE cdbs HOST bond + CMD mac_package RUN "/Users/iraf/daily_build/mac_package/make_packages cdbs" + AFTER cdbs/local_copy + +TABLE cdbs HOST arzach + CMD ftp RUN "cdbs_arc" + AFTER init/* + + @@ -6,20 +6,27 @@ # arzach assembles all the source code TABLE assemble HOST arzach + + CMD svnsync RUN "assemble_svnsync" + AFTER init/* + AFTER OPT irafx_update + CMD dev.stsci_python RUN "assemble_stsci_python dev" AFTER init/* + AFTER svnsync CMD dev.stsci_iraf RUN "assemble_stsci_iraf dev" AFTER init/* + AFTER svnsync CMD dev.axe RUN "assemble_axe dev" AFTER init/* + AFTER svnsync CMD dev.hstcal RUN "assemble_hstcal dev" AFTER init/* + AFTER svnsync - CMD nop RUN "sleep 1" - AFTER init/* #################### #################### @@ -73,7 +80,7 @@ TABLE build HOST thor arzach AFTER herbert:build/dev.stsci_iraf* TABLE build HOST cadeau - CMD dev.stsci_iraf_64hack RUN "build_stsci_iraf_64hack dev cadeau" + CMD dev.stsci_iraf_64hack RUN "build_stsci_iraf_64hack dev bond" AFTER bond:build/dev.stsci_iraf* # stsci_python documentation @@ -105,7 +112,7 @@ TABLE distribute HOST herbert thor arzach AFTER stamp/dev CMD dev.stsci_iraf RUN "synctool - stsci_iraf_dev" AFTER stamp/dev - CMD dev.hstcal RUN "synctool - hstcal_dev" + CMD dev.hstcal RUN "synctool - hstcaldev" AFTER stamp/dev CMD dev.motd RUN "synctool - irafdev/iraf/unix/hlib/motd" AFTER distribute/dev.iraf @@ -124,7 +131,7 @@ TABLE distribute HOST arzach AFTER stamp/dev CMD jwcalibdev.stsci_iraf RUN "synctool jwcalibdev: stsci_iraf_dev" AFTER stamp/dev - CMD jwcalibdev.hstcal RUN "synctool jwcalibdev: hstcal_dev" + CMD jwcalibdev.hstcal RUN "synctool jwcalibdev: hstcaldev" AFTER stamp/dev CMD jwcalibdev.motd RUN "synctool jwcalibdev: irafdev/iraf/unix/hlib/motd" AFTER jwcalibdev.iraf @@ -138,7 +145,7 @@ TABLE distribute_other HOST arzach AFTER stamp/dev CMD goods.stsci_iraf RUN "synctool goods12: stsci_iraf_dev" AFTER stamp/dev - CMD goods.hstcal RUN "synctool goods12: hstcal_dev" + CMD goods.hstcal RUN "synctool goods12: hstcaldev" AFTER stamp/dev CMD goods.motd RUN "synctool goods12: irafdev/iraf/unix/hlib/motd" AFTER goods.iraf @@ -152,7 +159,7 @@ TABLE distribute_other HOST arzach AFTER stamp/dev CMD witserv1.stsci_iraf RUN "synctool witserv1: stsci_iraf_dev" AFTER stamp/dev - CMD witserv1.hstcal RUN "synctool witserv1: hstcal_dev" + CMD witserv1.hstcal RUN "synctool witserv1: hstcaldev" AFTER stamp/dev CMD witserv1.motd RUN "synctool witserv1: irafdev/iraf/unix/hlib/motd" AFTER witserv1.iraf @@ -166,7 +173,7 @@ TABLE distribute_other HOST arzach AFTER stamp/dev CMD dmsinsvm.stsci_iraf RUN "synctool dmsinsvm: stsci_iraf_dev" AFTER stamp/dev - CMD dmsinsvm.hstcal RUN "synctool dmsinsvm: hstcal_dev" + CMD dmsinsvm.hstcal RUN "synctool dmsinsvm: hstcaldev" AFTER stamp/dev CMD dmsinsvm.motd RUN "synctool dmsinsvm: irafdev/iraf/unix/hlib/motd" AFTER dmsinsvm.iraf @@ -180,7 +187,7 @@ TABLE distribute_other HOST thor AFTER stamp/dev CMD udf1.stsci_iraf RUN "synctool udf1: stsci_iraf_dev" AFTER stamp/dev - CMD udf1.hstcal RUN "synctool udf1: hstcal_dev" + CMD udf1.hstcal RUN "synctool udf1: hstcaldev" AFTER stamp/dev CMD udf1.motd RUN "synctool udf1: irafdev/iraf/unix/hlib/motd" AFTER udf1.iraf @@ -194,7 +201,7 @@ TABLE distribute_other HOST thor AFTER stamp/dev CMD royal.stsci_iraf RUN "synctool royal: stsci_iraf_dev" AFTER stamp/dev - CMD royal.hstcal RUN "synctool royal: hstcal_dev" + CMD royal.hstcal RUN "synctool royal: hstcaldev" AFTER stamp/dev CMD royal.motd RUN "synctool royal: irafdev/iraf/unix/hlib/motd" AFTER royal.iraf @@ -0,0 +1,46 @@ +#!/bin/sh + +n=3 + +rm -rf build + +p=`which python` + +case "$p" +in +/usr/stsci/*) : ;; +*) + echo why is python set to $p + echo path = $PATH + exit 1 + ;; +esac + +unset PYTHONPATH + +there=/ssbwebv1/data2/steuermann/s$n + +rm -rf $there + +python setup.py -q install --home $there + +rm -f /eng/ssb/websites/ssb/steuermann/s$n.cgi + +ln -s $there/bin/steuermann_report.cgi /eng/ssb/websites/ssb/steuermann/s$n.cgi + +if grep -q s$n.cgi /eng/ssb/websites/ssb/index.html +then + : +else + id=/eng/ssb/websites/ssb + + sed 's?<\!--STEUERMANN-->?<\!--STEUERMANN--><a href="steuermann/s'$n'.cgi">s'$n'</a> <br> ?' < $id/index.html > tmp + + cp tmp $id/index.html + ls -l $id/index.html + +fi + +chgrp -R ssb $there + + @@ -1,12 +1,6 @@ -TABLE init HOST bond cadeau arzach thor herbert +TABLE init HOST bond cadeau arzach thor herbert ssb CMD sendscripts LOCAL "/eng/ssb/auto/steuermann_scripts/init_sendscripts %(hostname)s %(workdir)s" CMD sysstat RUN "sysstat" AFTER sendscripts -TABLE init HOST arzach - CMD svnsync RUN "assemble_svnsync" - AFTER OPT init/irafx_update - AFTER sendscripts - AFTER sysstat - diff --git a/scripts/smc b/scripts/smc index 3937855..0fc03f7 100644 --- a/scripts/smc +++ b/scripts/smc @@ -1,4 +1,7 @@ #! python -import steuermann.run_all as run_all +import sys +STEUERMANN_DIR_HERE +sys.path.insert(0, addpath) +import steuermann.run_all as run_all run_all.main() diff --git a/scripts/smcron b/scripts/smcron new file mode 100644 index 0000000..25a43b0 --- /dev/null +++ b/scripts/smcron @@ -0,0 +1,7 @@ +#! python +import sys +STEUERMANN_DIR_HERE +sys.path.insert(0, addpath) + +import steuermann.run_cron as run_cron +run_cron.main() diff --git a/scripts/steuermann_report.cgi b/scripts/steuermann_report.cgi index abf371b..6953b0d 100644 --- a/scripts/steuermann_report.cgi +++ b/scripts/steuermann_report.cgi @@ -6,6 +6,7 @@ import os import sys import re import datetime +import pandokia.text_table STEUERMANN_DIR_HERE @@ -18,6 +19,25 @@ cgitb.enable() form = cgi.FieldStorage(keep_blank_values=1) cginame = os.getenv("SCRIPT_NAME") +permission_modify=1 + +html_header='''Content-type: text/html + +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN"> +<HTML> +<HEAD> +<TITLE>%(title)s</TITLE> +</HEAD> +<BODY> +''' + +html_trailer=''' +</BODY> +</HTML> +''' + +########## + def sqltime(arg) : if arg is None : return None @@ -36,30 +56,169 @@ def sqltime(arg) : ########## + +def normalize_run_name(db, name) : + + if name == 'daily_latest' : + c = db.execute("SELECT max(run) FROM sm_runs WHERE run like 'daily_%'") + run, = c.fetchone() + return run + + return name + +########## # if no action specified, show the list of runs # -if not 'action' in form : - print 'content-type: text/html' - print '' +if 'action' in form : + action = form['action'].value +else : + action = 'index' + +if action == 'index' : + print html_header % { 'title' : 'Steuermann' } + print "<a href=%s?action=runs>runs</a><br>"%cginame + print "<a href=%s?action=crons>crons</a><br>"%cginame + print html_trailer + sys.exit(0) + + +########## +# list the runs + +elif action == 'crons' : + print html_header % { 'title' : 'Steuermann List' } + db = steuermann.config.open_db() + c = db.cursor() + + c.execute("SELECT host, name, decollision, start_time, end_time, duration, status FROM sm_crons ORDER BY start_time desc") + + tt = pandokia.text_table.text_table() + tt.set_html_table_attributes("border=1") + tt.define_column('host') + tt.define_column('name') + tt.define_column('start_time') + tt.define_column('end_time') + tt.define_column('duration') + tt.define_column('status') + if permission_modify : + tt.define_column('delete') + + link="%s?action=%s&host=%s&name=%s&decol=%s" + for host, name, decollision, start_time, end_time, duration, status in c : + row = tt.get_row_count() + short_name = name.split('--')[0] + tt.set_value(row, 'host', host ) + tt.set_value(row, 'name', text=short_name, link=link%(cginame,'cronlog',host,name,decollision) ) + tt.set_value(row, 'start_time', start_time) + if end_time is None : + end_time = '' + if end_time.startswith(start_time[0:11]) : + tt.set_value(row, 'end_time', end_time[11:]) + else : + tt.set_value(row, 'end_time', end_time) + tt.set_value(row, 'duration', duration) + tt.set_value(row, 'status', status) + if permission_modify : + tt.set_value(row, 'delete', 'arf') + + print tt.get_html() + print html_trailer + sys.exit(0) + +elif action == 'runs' : + print html_header % { 'title' : 'Steuermann List' } + print 'sort: ' + for x in ( 'name_asc', 'name_desc', 'time_asc', 'time_desc' ) : + print "<a href=%s?action=runs&order=%s>%s</a>"%(cginame,x,x) + print "<br><br>" db = steuermann.config.open_db() c = db.cursor() - c.execute('SELECT DISTINCT run FROM sm_status ORDER BY run DESC') - for run, in c : - print "<a href=%s?action=status&run=%s>%s</a><br>"%(cginame, run, run) + order='create_time DESC' + + if 'order' in form : + x = form['order'].value + if x == 'name_asc' : + order = 'run ASC' + elif x == 'name_desc' : + order = 'run DESC' + elif x == 'time_asc' : + order ='create_time ASC' + elif x == 'time_desc' : + order ='create_time DESC' + + c.execute('SELECT DISTINCT run, create_time FROM sm_runs ORDER BY %s'%order) + print "<table>" + for run, create_time in c : + print "<tr>" + print "<td>" + print "<a href=%s?action=status&run=%s>%s</a>"%(cginame, run, run) + print "</td><td>" + if permission_modify : + print "<a href=%s?action=delete&run=%s>delete</a>"%(cginame, run) + print "</td>" + print "</tr>" + print "</table>" + print html_trailer sys.exit(0) -action = form['action'].value ########## # status means show the status of a particular run # -if action == 'status' : +elif action == 'delete' : + if permission_modify : + print 'content-type: text/plain\n' + in_run = form['run'].value + db = steuermann.config.open_db() + in_run = normalize_run_name(db,in_run) + c = db.cursor() + c1 = db.cursor() + print "RUN=",in_run + c.execute("SELECT run FROM sm_runs WHERE run LIKE ?",(in_run,)) + for run, in c : + print "echo run ",run + filename = '%s/run/%s'%(steuermann.config.logdir,run) + print "rm -rf ",filename + c.execute("DELETE FROM sm_runs WHERE run LIKE ?",(in_run,)) + db.commit() + sys.exit(0) + +########## +# + +elif action == 'status' : db = steuermann.config.open_db() import steuermann.report steuermann.report.cginame = cginame - print 'content-type: text/html' + print html_header % { 'title' : 'Steuermann Status' } print '' run = form['run'].value + run = normalize_run_name(db,run) print steuermann.report.report_html( db, run, info_callback=steuermann.report.info_callback_gui ) + print html_trailer + sys.exit(0) + + +elif action == 'cronlog' : + print 'content-type: text/plain' + print '' + host = form['host'].value + name = form['name'].value + decol = form['decol'].value + db = steuermann.config.open_db() + c = db.cursor() + + c.execute("SELECT host, name, decollision, start_time, end_time, status, logfile FROM sm_crons WHERE host = ? AND name = ? and decollision = ?", (host, name, decol) ) + for host, name, decollision, start_time, end_time, status, logfile in c : + print "----------" + print "%s: %s\n"%(host,name) + print "decol=%s"%decollision + print start_time + print end_time + print "status=",status + print "----------" + f=open(steuermann.config.logdir + '/cron/' + logfile,"r") + sys.stdout.write(f.read()) + f.close() sys.exit(0) ########## @@ -103,7 +262,7 @@ elif action == 'log' : for x in [ ' ' + x for x in notes.split('\n') ] : print x print "" - filename = '%s/%s/%s:%s.%s.log'%(steuermann.config.logdir,run,host,table,cmd) + filename = '%s/run/%s/%s:%s.%s.log'%(steuermann.config.logdir,run,host,table,cmd) try : f=open(filename,'r') except IOError: @@ -124,7 +283,7 @@ elif action == 'log' : # info means show information about the system # elif action == 'info' : - print 'content-type: text/html\n' + print html_header % { 'title': 'Steuermann Info' } print 'db credentials: ',steuermann.config.db_creds,'<br>' print 'logdir: ',steuermann.config.logdir,'<br>' db = steuermann.config.open_db() @@ -135,11 +294,12 @@ elif action == 'info' : cur.execute("select count(*) from sm_runs") l = cur.fetchone() print "runs: %s\n"%l[0],'<br>' + print html_trailer sys.exit(0) ########## -print 'content-type: text/html' +print html_header print '' print 'no recognized action?' - +print html_trailer @@ -1,5 +1,38 @@ import distutils.core +long_desc = ''' +Steuermann is a control system for continuous integration. You write +one or more config files defining processes to run on various computers +along with dependencies. + +For example, you can declare that task A runs on computer X, but only +after task B runs (to completion) on computer Y. + +You can invoke steuermann in automatic mode to run all the processes in +the necessary order. This is the way to perform automatic builds/tests. + +You can invoke steuermann manually to select a subset of processes +to execute. This is a way to repeat steps without running everything +again, for example to repeat a failed portion of an automatic build/test +sequence. + +''' + +classifiers = [ + 'Development Status :: 3 - Alpha', + 'Environment :: Console', + 'Environment :: Web Environment', + 'Intended Audience :: Developers', + 'Intended Audience :: System Administrators', + 'License :: OSI Approved :: BSD License', + 'Natural Language :: English', + 'Operating System :: OS Independent', + 'Programming Language :: Python :: 2', + 'Topic :: Software Development :: Build Tools', + 'Topic :: System :: Distributed Computing', + ] + +# f=open('steuermann/__init__.py','r') for x in f : if x.startswith('__version__') : @@ -7,22 +40,25 @@ for x in f : break f.close() -print version - -command_list = [ 'smc', 'steuermann_report.cgi' ] +# +command_list = [ 'smc', 'smcron', 'steuermann_report.cgi' ] use_usr_bin_env = [ ] dir_set = 'addpath = "%s"\n' args = { - 'version' : "2.0dev", + 'name': 'steuermann', + 'version' : version, 'description' : "Steuermann Continuous Integration Control System", + 'long_description': long_desc, 'author' : "Mark Sienkiewicz", - 'scripts' : ['scripts/smc', 'scripts/steuermann_report.cgi'], - 'package_dir' : { 'steuermann' : 'steuermann', }, + 'author_email' : "help@stsci.edu", 'url' : 'https://svn.stsci.edu/trac/ssb/etal/wiki/Steuermann', + 'scripts' : ['scripts/' + x for x in command_list ], + 'package_dir' : { 'steuermann' : 'steuermann', }, 'license': 'BSD', 'packages': [ 'steuermann' ], - 'package_data': { 'steuermann' : [ '*.sql', '*.ini', ], } + 'package_data': { 'steuermann' : [ '*.sql', '*.ini', ], }, + 'classifiers': classifiers, } d = distutils.core.setup( diff --git a/steuermann/db.sql b/steuermann/db.sql index b6d2ae6..99bc752 100644 --- a/steuermann/db.sql +++ b/steuermann/db.sql @@ -43,8 +43,31 @@ create unique index sm_status_idx1 on sm_status ( run, host, tablename, cmd ); -- table lists all run names in the system CREATE TABLE sm_runs ( - run VARCHAR(100) + run VARCHAR(100), + create_time VARCHAR(26) ); CREATE UNIQUE INDEX sm_runs_idx1 ON sm_runs(run); + +-- table lists scheduled cron events - sort of like the old sr +-- smcron name script +CREATE TABLE sm_crons ( + host VARCHAR, + -- what host we ran this on + name VARCHAR(100), + -- a descriptive name, not necessarily unique + decollision VARCHAR(10), + -- a cookie to make (host,name,decollision) unique + start_time VARCHAR(30), + end_time VARCHAR(30), + duration REAL, + -- + status VARCHAR(5), + -- exit status + logfile VARCHAR(1000) + -- log file name, relative to config logdir+'/cron/' + ); + +CREATE INDEX sm_crons_idx1 ON sm_crons( host, name, decollision ); + diff --git a/steuermann/hosts.ini b/steuermann/hosts.ini index f8c2e3b..abc114c 100644 --- a/steuermann/hosts.ini +++ b/steuermann/hosts.ini @@ -28,12 +28,15 @@ local=[ 'sh', '-c', '%(script)s' ] ; ; -q = quiet ; -x = do not forward X windows; prevents errors locking .Xauthority when lots of ssh come in at the same time -run=[ 'ssh', '-q', '-x', '%(hostname)s', ' cd %(workdir)s; set node=%(node)s; source bin/.steuermann.%(hostname)s; %(script)s; show_status $status ' ] +run=[ 'ssh', '-q', '-x', '%(hostname)s', ' cd %(workdir)s; setenv sm_node %(node)s; setenv sm_run %(runname)s; source bin/.steuermann.%(hostname)s; %(script)s; show_status $status ' ] like=all [mac:csh] like=linux:csh +[solaris:csh] +like=linux:csh + [windows:cmd] run=[ 'python', '-m', 'steuermann.windows_comm', '%(hostname)s', '%(script)s' ] like=all @@ -59,6 +62,12 @@ like=cadeau ; actual machines +[jwcalibdev] +hostname=jwcalibdev +like=linux:csh +workdir=/data1/iraf/steuermann +maxproc=32 + [herbert] hostname=herbert like=linux:csh @@ -77,6 +86,9 @@ like=linux:csh workdir=/arzach/data1/iraf/steuermann maxproc=4 +[localhost] +like=ssb + [ssb] hostname=ssbwebv1 like=linux:csh @@ -101,5 +113,18 @@ like=mac:csh workdir=/Users/iraf/work/steuermann maxproc=4 +[aten] +hostname=aten +like=solaris:csh +maxproc=1 +workdir=/aten/data2/iraf/steuermann + +[grail] +hostname=grail +like=solaris:csh +maxproc=1 +workdir=/data/grail1/iraf/steuermann + + ; There is a section [ALL] that is used with every machine name [ALL] diff --git a/steuermann/nodes.py b/steuermann/nodes.py index 0f24e36..476734a 100644 --- a/steuermann/nodes.py +++ b/steuermann/nodes.py @@ -212,6 +212,10 @@ class node(object) : self.finished = 0 self.running = 0 + # + self.wanted = 1 + self.skip = 1 + ##### # debug - make a string representation of all the nodes diff --git a/steuermann/report.py b/steuermann/report.py index cc90604..8d95d1f 100644 --- a/steuermann/report.py +++ b/steuermann/report.py @@ -11,18 +11,6 @@ import StringIO # maybe the output is html 3.2 - in any case, it is way simpler than # more recent standards. -html_header='''<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN"> -<HTML> -<HEAD> -<TITLE>%(title)s</TITLE> -</HEAD> -<BODY> -''' - -html_trailer=''' -</BODY> -</HTML> -''' # this will be reset by the cgi main program if we are in a real cgi cginame = 'arf.cgi' @@ -167,7 +155,6 @@ def report_text( db, run_name, info_callback = info_callback_status ) : def report_html( db, run_name, info_callback = info_callback_status, hlevel=1 ) : s = StringIO.StringIO() - s.write(html_header % { 'title' : run_name } ) s.write('<h%d>%s</h%d>\n'%(hlevel,run_name,hlevel)) hlevel = hlevel + 1 @@ -179,8 +166,6 @@ def report_html( db, run_name, info_callback = info_callback_status, hlevel=1 ) t = get_table( db, run_name, tablename, info_callback, showdepth=1 ) s.write(t.get_html()) - s.write(html_trailer) - return s.getvalue() # diff --git a/steuermann/run.py b/steuermann/run.py index af52a02..e83fca2 100644 --- a/steuermann/run.py +++ b/steuermann/run.py @@ -3,12 +3,32 @@ run processes asynchronously on various machines, with a callback on process exit. ''' +# to do someday: +# +# This feature should really be broken into 3 parts: +# - remotely execute on another machine +# - track concurrent execution +# - reserve resource usage +# +# To start a process, ask for a resource reservation. (Currently, the +# only resource we track is CPUs.) If we don't get a reservation, we +# don't run right away. +# +# If we do, use the remote exec to run the process on the target machine. +# This is the part that knows hosts.ini. (We also use hosts.ini to declare +# resource availability.) +# +# When the process finishes, release the resource reservation. +# + import subprocess import time import datetime import os +import os.path import traceback import sys +import errno import ConfigParser @@ -56,7 +76,7 @@ class runner(object): ##### # start a process - def run( self, node, run_name, no_run = False ): + def run( self, node, run_name, no_run = False, logfile_name = None ): try : try : @@ -72,14 +92,15 @@ class runner(object): n = int(self.howmany.get(hostname,0)) if n >= int(args['maxproc']) : - print "decline to run %s - %d other already running"%(node.name,n) + # print "decline to run %s - %d other already running"%(node.name,n) return False n = n + 1 self.howmany[hostname] = n - print "running %s %s %d"%(hostname,node.name, n) + # print "running %s %s %d"%(hostname,node.name, n) else : - print "running %s %s no maxproc"%(hostname, node.name) + # print "running %s %s no maxproc"%(hostname, node.name) + pass if debug : print "run",node.name @@ -96,6 +117,7 @@ class runner(object): table=node.table, cmd=node.cmd, node=node.name, + runname=run_name, ) if debug : @@ -122,15 +144,20 @@ class runner(object): if debug : print "RUN",run - # make sure the log directory is there - logdir= self.logdir + "/%s"%run_name - try : - os.makedirs(logdir) - except OSError: - pass + if logfile_name is None : + # make sure the log directory is there + logdir= self.logdir + "/%s"%(run_name) - # create a name for the log file, but do not use / in the name - logfile_name = "%s/%s.log"%( logdir, node.name.replace('/','.') ) + # create a name for the log file, but do not use / in the name + logfile_name = "%s/%s.log"%( logdir, node.name.replace('/','.') ) + + try : + os.makedirs( os.path.dirname(logfile_name) ) + except OSError, e : + if e.errno == errno.EEXIST : + pass + else : + raise # open the log file, write initial notes logfile=open(logfile_name,"w") @@ -142,6 +169,8 @@ class runner(object): run = [ 'echo', 'no_run - node=', node.name ] # start running the process + if debug : + print "RUN",run p = subprocess.Popen(args=run, stdout=logfile, stderr=subprocess.STDOUT, @@ -177,7 +206,8 @@ class runner(object): n = self.howmany[hostname] - 1 self.howmany[hostname] = n - print "finish %s %s %d"%(hostname,node_name,n) + if debug : + print "finish %s %s %d"%(hostname,node_name,n) # note the termination of the process at the end of the log file logfile = self.all_procs[node_name].logfile @@ -270,7 +300,6 @@ class runner(object): d1 = d1.copy() d1.update(d) d = d1 - print d del d['like'] # default hostname is the name from the section header diff --git a/steuermann/run_all.py b/steuermann/run_all.py index f670abe..1e179a2 100644 --- a/steuermann/run_all.py +++ b/steuermann/run_all.py @@ -11,6 +11,7 @@ import datetime import run import report import nodes +import getpass import steuermann.config @@ -22,6 +23,9 @@ try : except ImportError : readline = None + +username=getpass.getuser() + ##### def main() : @@ -68,7 +72,7 @@ def main() : if '-r' in opt : run_name = opt['-r'] else : - run_name = str(datetime.datetime.now()).replace(' ','_') + run_name = "user_%s_%s"%(username,str(datetime.datetime.now()).replace(' ','_')) db = steuermann.config.open_db() @@ -348,7 +352,7 @@ def print_pre(who, xnodes, depth) : def register_database(db, run, xnodes ) : c = db.cursor() - c.execute('INSERT INTO sm_runs ( run ) VALUES ( ? )', ( run, ) ) + c.execute('INSERT INTO sm_runs ( run, create_time ) VALUES ( ?, ? )', ( run, str(datetime.datetime.now()).replace(' ','_')) ) c = db.cursor() for x in xnodes : @@ -363,14 +367,17 @@ def register_database(db, run, xnodes ) : def run_all(xnodes, run_name, db) : + for x in xnodes : + x = xnodes[x] + x.finished = 0 + x.running = 0 + x.wanted = 1 + x.skip = 0 + register_database(db, run_name, xnodes) - runner = run.runner( xnodes ) + runner = run.runner( xnodes, steuermann.config.logdir ) - for x in xnodes : - xnodes[x].finished = 0 - xnodes[x].running = 0 - xnodes[x].wanted = 1 while 1 : ( keep_running, no_sleep ) = run_step( runner, xnodes, run_name, db ) diff --git a/steuermann/run_cron.py b/steuermann/run_cron.py new file mode 100644 index 0000000..39d5be7 --- /dev/null +++ b/steuermann/run_cron.py @@ -0,0 +1,82 @@ +import os +import sys +import time +import steuermann.config +import steuermann.run +import datetime + +# bugs: +# - this is a hideous hack that works around run.py not being designed for this +# - the work directory on the target machine is poorly named +# + +class fakenode(object): + pass + +def main() : + + if len(sys.argv) < 4 : + print "smcron host name command" + print "(from ",sys.argv,")" + sys.exit(1) + + host = sys.argv[1] + name = sys.argv[2] + script = ' '.join(sys.argv[3:]) + + db = steuermann.config.open_db() + + c = db.cursor() + + start_time = datetime.datetime.now() + day, tod = str(start_time).split(' ') + + name = name + '--' + tod + + logfile = '%s/%s.%s' % (day, host, name) + + # something to reduce the possibility of a collision to very small - in this case, we assume that + # we can't start multiple processes at the same time from the same process number. i.e. the + # pid can not wrap in less than 1 clock tick + decollision = os.getpid() + + c.execute("INSERT INTO sm_crons ( host, name, start_time, logfile, decollision ) VALUES ( ?, ?, ?, ?, ? )", + ( host, name, str(start_time), logfile, decollision ) ) + db.commit() + + # this is a hideous hack to plug on to an interface that was not designed for this + + node = fakenode() + node.host = host + node.name = 'cron.' + name + node.table = None + node.cmd = None + node.script = script + if host == 'localhost' : + node.script_type = 'l' # local + else : + node.script_type = 'r' # remote + + runner = steuermann.run.runner( nodes = { node.name : node }, logdir=None ) + runner.run( node=node, run_name='', logfile_name = steuermann.config.logdir + '/cron/' + logfile ) + + n = 0.1 + while 1 : + exited = runner.poll() + if exited : + break + if n < 2.0 : + n = n * 2.0 + time.sleep(n) + + status = exited[1] + + end_time = datetime.datetime.now() + td = end_time - start_time + + # only 1 decimal place because we don't poll often enough to make more reasonable. + td = "%.1f" % ( td.microseconds/1e6 + td.seconds + td.days * 24 * 3600 ) + c.execute("UPDATE sm_crons SET end_time = ?, status = ?, duration = ? WHERE host = ? AND name = ? ", + (str(end_time), status, td, host, name ) ) + db.commit() + diff --git a/x_build.sm b/x_build.sm new file mode 100644 index 0000000..d5702ac --- /dev/null +++ b/x_build.sm @@ -0,0 +1,80 @@ +## TODO: +## add builds on ssbwebv1 for those things that we actually care about + +#################### +#################### + +# arzach assembles all the source code +TABLE assemble HOST arzach + + CMD svnsync RUN "assemble_svnsync" + AFTER init/* + AFTER OPT irafx_update + + CMD x.stsci_python RUN "assemble_stsci_python x" + AFTER init/* + AFTER svnsync + + CMD x.stsci_iraf RUN "assemble_stsci_iraf x" + AFTER init/* + AFTER svnsync + + CMD x.axe RUN "assemble_axe x" + AFTER init/* + AFTER svnsync + + CMD x.hstcal RUN "assemble_hstcal x" + AFTER init/* + AFTER svnsync + + +#################### +#################### + +# install stsci_python into default environment +# build hstcal +# - everywhere +TABLE build HOST herbert thor arzach bond cadeau + CMD x.py2.7 RUN "build_stsci_python x 2.7" + AFTER init/* + AFTER *:assemble/x.stsci_python + CMD x.hstcal RUN "build_hstcal x" + AFTER init/* + AFTER *:assemble/x.hstcal + +# stsdas and friends +# - 32 bit only +TABLE build HOST herbert bond + CMD x.axe RUN "build_axe x" + AFTER init/* + AFTER *:assemble/x.axe + + CMD x.stsci_iraf RUN "build_stsci_iraf x" + AFTER init/* + AFTER *:assemble/x.stsci_iraf + AFTER build/x.axe + + CMD x.stsci_iraf_log RUN "build_stsci_iraf_log x" + AFTER init/* + AFTER build/x.stsci_iraf + + CMD x.stsci_iraf_help RUN "build_stsci_iraf_help x" + AFTER init/* + AFTER build/x.stsci_iraf + + +# stsdas for 64 bit machines - get it from a related 32 bit system +TABLE build HOST thor arzach + CMD x.stsci_iraf_64hack RUN "build_stsci_iraf_64hack x herbert" + AFTER herbert:build/x.stsci_iraf* + +TABLE build HOST cadeau + CMD x.stsci_iraf_64hack RUN "build_stsci_iraf_64hack x bond" + AFTER bond:build/x.stsci_iraf* + + +# stamp the IRAF banner file when the builds are complete +TABLE stamp HOST herbert thor arzach bond cadeau + CMD x RUN "build_stamp x" + AFTER build/* + diff --git a/x_dist.sm b/x_dist.sm new file mode 100644 index 0000000..5218a9b --- /dev/null +++ b/x_dist.sm @@ -0,0 +1,104 @@ + +# regular distributions + +TABLE distribute HOST herbert thor arzach + CMD x.iraf RUN "synctool - irafx" + AFTER stamp/x + CMD x.pyssg RUN "synctool - pyssgx" + AFTER stamp/x + CMD x.stsci_iraf RUN "synctool - stsci_iraf_x" + AFTER stamp/x + CMD x.hstcal RUN "synctool - hstcal_x" + AFTER stamp/x + CMD x.motd RUN "synctool - irafx/iraf/unix/hlib/motd" + AFTER distribute/x.iraf + +TABLE distribute HOST bond cadeau + CMD irafx.pkg RUN "cd $HOME/daily_build/mac_package; ./clean ; ./build x " AFTER stamp/x + CMD irafx.dmg RUN "cd $HOME/daily_build/mac_package; ./distribute x" AFTER irafx.pkg + +# wads of special cases + +# jwcalibdev has local disk - some day it may do its own builds +TABLE distribute HOST arzach + CMD jwcalibdev.iraf RUN "synctool jwcalibdev: irafx" + AFTER stamp/x + CMD jwcalibdev.pyssg RUN "synctool jwcalibdev: pyssgx" + AFTER stamp/x + CMD jwcalibdev.stsci_iraf RUN "synctool jwcalibdev: stsci_iraf_x" + AFTER stamp/x + CMD jwcalibdev.hstcal RUN "synctool jwcalibdev: hstcalx" + AFTER stamp/x + CMD jwcalibdev.motd RUN "synctool jwcalibdev: irafx/iraf/unix/hlib/motd" + AFTER jwcalibdev.iraf + +# goods - has RHE 5 only now + +TABLE distribute_other HOST arzach + CMD goods.iraf RUN "synctool goods12: irafx" + AFTER stamp/x + CMD goods.pyssg RUN "synctool goods12: pyssgx" + AFTER stamp/x + CMD goods.stsci_iraf RUN "synctool goods12: stsci_iraf_x" + AFTER stamp/x + CMD goods.hstcal RUN "synctool goods12: hstcalx" + AFTER stamp/x + CMD goods.motd RUN "synctool goods12: irafx/iraf/unix/hlib/motd" + AFTER goods.iraf + +# witserv1 - who are these guys? + +TABLE distribute_other HOST arzach + CMD witserv1.iraf RUN "synctool witserv1: irafx" + AFTER stamp/x + CMD witserv1.pyssg RUN "synctool witserv1: pyssgx" + AFTER stamp/x + CMD witserv1.stsci_iraf RUN "synctool witserv1: stsci_iraf_x" + AFTER stamp/x + CMD witserv1.hstcal RUN "synctool witserv1: hstcalx" + AFTER stamp/x + CMD witserv1.motd RUN "synctool witserv1: irafx/iraf/unix/hlib/motd" + AFTER witserv1.iraf + +# dmsinsvm - have a pipeline and irafx/irafdev on the same machine for INS + +TABLE distribute_other HOST arzach + CMD dmsinsvm.iraf RUN "synctool dmsinsvm: irafx" + AFTER stamp/x + CMD dmsinsvm.pyssg RUN "synctool dmsinsvm: pyssgx" + AFTER stamp/x + CMD dmsinsvm.stsci_iraf RUN "synctool dmsinsvm: stsci_iraf_x" + AFTER stamp/x + CMD dmsinsvm.hstcal RUN "synctool dmsinsvm: hstcalx" + AFTER stamp/x + CMD dmsinsvm.motd RUN "synctool dmsinsvm: irafx/iraf/unix/hlib/motd" + AFTER dmsinsvm.iraf + +# UDF - another funded project with their own machines + +TABLE distribute_other HOST thor + CMD udf1.iraf RUN "synctool udf1: irafx" + AFTER stamp/x + CMD udf1.pyssg RUN "synctool udf1: pyssgx" + AFTER stamp/x + CMD udf1.stsci_iraf RUN "synctool udf1: stsci_iraf_x" + AFTER stamp/x + CMD udf1.hstcal RUN "synctool udf1: hstcal_x" + AFTER stamp/x + CMD udf1.motd RUN "synctool udf1: irafx/iraf/unix/hlib/motd" + AFTER udf1.iraf + +# royal - a beowulf cluster + +TABLE distribute_other HOST thor + CMD royal.iraf RUN "synctool royal: irafx" + AFTER stamp/x + CMD royal.pyssg RUN "synctool royal: pyssgx" + AFTER stamp/x + CMD royal.stsci_iraf RUN "synctool royal: stsci_iraf_x" + AFTER stamp/x + CMD royal.hstcal RUN "synctool royal: hstcalx" + AFTER stamp/x + CMD royal.motd RUN "synctool royal: irafx/iraf/unix/hlib/motd" + AFTER royal.iraf + |