diff options
| author | sienkiew <sienkiew@d34015c8-bcbb-4646-8ac8-8ba5febf221d> | 2012-01-18 17:36:05 -0500 |
|---|---|---|
| committer | sienkiew <sienkiew@d34015c8-bcbb-4646-8ac8-8ba5febf221d> | 2012-01-18 17:36:05 -0500 |
| commit | a5afe5f657f5812ea95c5eb476eb355de6afa5a9 (patch) | |
| tree | f14049a9056e660abf736073ae14a7221011fca0 | |
| parent | c7082d39ae1f653fc33383d9e093d54b00022fd5 (diff) | |
| download | steuermann-a5afe5f657f5812ea95c5eb476eb355de6afa5a9.tar.gz | |
new sm files for nightly builds
IMPORT feature in specification language
checkpointing of various uncommitted changes
git-svn-id: https://svn.stsci.edu/svn/ssb/etal/steuermann/trunk@538 d34015c8-bcbb-4646-8ac8-8ba5febf221d
38 files changed, 717 insertions, 64 deletions
@@ -1,4 +1,4 @@ -TABLE cdbs HOST bond cadeau +TABLE cdbs HOST bond cadeau banana CMD local_copy RUN "cdbs_copy" AFTER init/* @@ -1,32 +1,10 @@ ## TODO: ## add builds on ssbwebv1 for those things that we actually care about -#################### -#################### - -# arzach assembles all the source code -TABLE assemble HOST arzach - - CMD svnsync RUN "assemble_svnsync" - AFTER init/* - AFTER OPT irafx_update - - CMD dev.stsci_python RUN "assemble_stsci_python dev" - AFTER init/* - AFTER svnsync - - CMD dev.stsci_iraf RUN "assemble_stsci_iraf dev" - AFTER init/* - AFTER svnsync - - CMD dev.axe RUN "assemble_axe dev" - AFTER init/* - AFTER svnsync - - CMD dev.hstcal RUN "assemble_hstcal dev" - AFTER init/* - AFTER svnsync +TABLE build HOST arzach + CMD dev.astrolib RUN "build_astrolib" + AFTER arzach:assemble/dev.astrolib #################### #################### @@ -34,7 +12,7 @@ TABLE assemble HOST arzach # install stsci_python into default environment # build hstcal # - everywhere -TABLE build HOST herbert thor arzach bond cadeau +TABLE build HOST herbert thor arzach bond cadeau banana CMD dev.py2.7 RUN "build_stsci_python dev 2.7" AFTER init/* AFTER *:assemble/dev.stsci_python @@ -79,7 +57,7 @@ TABLE build HOST thor arzach CMD dev.stsci_iraf_64hack RUN "build_stsci_iraf_64hack dev herbert" AFTER herbert:build/dev.stsci_iraf* -TABLE build HOST cadeau +TABLE build HOST cadeau banana CMD dev.stsci_iraf_64hack RUN "build_stsci_iraf_64hack dev bond" AFTER bond:build/dev.stsci_iraf* @@ -96,7 +74,7 @@ TABLE build HOST thor AFTER build/dev.py2.7 # stamp the IRAF banner file when the builds are complete -TABLE stamp HOST herbert thor arzach bond cadeau +TABLE stamp HOST herbert thor arzach bond cadeau banana CMD dev RUN "build_stamp dev" AFTER build/* @@ -117,7 +95,7 @@ TABLE distribute HOST herbert thor arzach CMD dev.motd RUN "synctool - irafdev/iraf/unix/hlib/motd" AFTER distribute/dev.iraf -TABLE distribute HOST bond cadeau +TABLE distribute HOST bond cadeau banana CMD irafdev.pkg RUN "cd $HOME/daily_build/mac_package; ./clean ; ./build dev " AFTER stamp/dev CMD irafdev.dmg RUN "cd $HOME/daily_build/mac_package; ./distribute dev -" AFTER irafdev.pkg @@ -211,7 +189,7 @@ TABLE distribute_other HOST thor # This structure is meant to serialize the regtests so that nothing else # is going on at the same time. The regtests can consume all the CPUs. -TABLE regtest HOST herbert thor arzach bond cadeau +TABLE regtest HOST herbert thor arzach bond cadeau banana CMD okify RUN "test_okify" AFTER distribute/* CMD dev.test2.7 RUN "test_rt dev 2.7" @@ -1,6 +1,6 @@ #!/bin/sh -n=3 +n=4 rm -rf build @@ -1,6 +1,5 @@ -TABLE init HOST bond cadeau arzach thor herbert ssb aten grail jwcalibdev +TABLE init HOST banana bond cadeau arzach thor herbert ssb aten jwcalibdev etcbrady CMD sendscripts LOCAL "/eng/ssb/auto/steuermann_scripts/init_sendscripts %(hostname)s %(workdir)s" CMD sysstat RUN "sysstat" AFTER sendscripts - diff --git a/pyetc.sm b/pyetc.sm new file mode 100644 index 0000000..d1b4419 --- /dev/null +++ b/pyetc.sm @@ -0,0 +1,20 @@ +#################### +#################### + +TABLE pyetc HOST etcbrady + + CMD trunk RUN "/data1/nightly/trunk/test-ppsetc -d trunk " + AFTER init/* + + CMD refactored RUN "/data1/nightly/refactored/test-ppsetc -d refactored " + AFTER trunk + AFTER init/* + + CMD jwst RUN "/data1/nightly/jwpetc_dev/test-ppsetc -d" + AFTER refactored + AFTER init/* + + CMD rt RUN "/data1/nightly/rt/rt.csh 7" + AFTER jwst + AFTER init/* + diff --git a/scripts/steuermann_report.cgi b/scripts/steuermann_report.cgi index 2c07c70..adb7f0b 100644 --- a/scripts/steuermann_report.cgi +++ b/scripts/steuermann_report.cgi @@ -19,7 +19,7 @@ cgitb.enable() form = cgi.FieldStorage(keep_blank_values=1) cginame = os.getenv("SCRIPT_NAME") -permission_modify=1 +permission_modify=0 html_header='''Content-type: text/html diff --git a/sm/A_astrolib.sm b/sm/A_astrolib.sm new file mode 100644 index 0000000..9da7ffc --- /dev/null +++ b/sm/A_astrolib.sm @@ -0,0 +1,5 @@ +IMPORT "init.sm" +IMPORT "assemble_common.sm" +IMPORT "assemble_astrolib.sm" +IMPORT "build_astrolib.sm" + diff --git a/sm/A_cdbs.sm b/sm/A_cdbs.sm new file mode 100644 index 0000000..f7b3c11 --- /dev/null +++ b/sm/A_cdbs.sm @@ -0,0 +1,2 @@ +IMPORT "init.sm" +IMPORT "cdbs.sm" diff --git a/sm/A_dev.sm b/sm/A_dev.sm new file mode 100644 index 0000000..8ada106 --- /dev/null +++ b/sm/A_dev.sm @@ -0,0 +1,3 @@ +IMPORT "B_dev.sm" +IMPORT "test_dev.sm" + diff --git a/sm/A_pyetc.sm b/sm/A_pyetc.sm new file mode 100644 index 0000000..20be13d --- /dev/null +++ b/sm/A_pyetc.sm @@ -0,0 +1 @@ +IMPORT "pyetc.sm" diff --git a/sm/A_x.sm b/sm/A_x.sm new file mode 100644 index 0000000..217f77d --- /dev/null +++ b/sm/A_x.sm @@ -0,0 +1,11 @@ +IMPORT "B_x.sm" +IMPORT "test_x.sm" + +TABLE assemble HOST arzach + CMD irafx_update RUN "/eng/ssb/auto/prog/irafx_update " + AFTER init/* + + CMD irafx_age RUN "/eng/ssb/auto/prog/irafx_age.py" + AFTER irafx_update + AFTER OPT svnsync + diff --git a/sm/A_xb.sm b/sm/A_xb.sm new file mode 100644 index 0000000..d8c1c7f --- /dev/null +++ b/sm/A_xb.sm @@ -0,0 +1,2 @@ +IMPORT "B_x.sm" +IMPORT "test_x.sm" diff --git a/sm/B_dev.sm b/sm/B_dev.sm new file mode 100644 index 0000000..8464322 --- /dev/null +++ b/sm/B_dev.sm @@ -0,0 +1,5 @@ +IMPORT "init.sm" +IMPORT "assemble_common.sm" +IMPORT "assemble_dev.sm" +IMPORT "build_dev.sm" +IMPORT "dist_dev.sm" diff --git a/sm/B_x.sm b/sm/B_x.sm new file mode 100644 index 0000000..a0183a7 --- /dev/null +++ b/sm/B_x.sm @@ -0,0 +1,5 @@ +IMPORT "init.sm" +IMPORT "assemble_common.sm" +IMPORT "assemble_x.sm" +IMPORT "build_x.sm" +IMPORT "dist_x.sm" diff --git a/sm/assemble_astrolib.sm b/sm/assemble_astrolib.sm new file mode 100644 index 0000000..16a1494 --- /dev/null +++ b/sm/assemble_astrolib.sm @@ -0,0 +1,6 @@ +TABLE assemble HOST arzach + + CMD dev.astrolib RUN "assemble_astrolib" + AFTER init/* + AFTER svnsync + diff --git a/sm/assemble_common.sm b/sm/assemble_common.sm new file mode 100644 index 0000000..03f36db --- /dev/null +++ b/sm/assemble_common.sm @@ -0,0 +1,10 @@ +#################### +#################### + +# arzach assembles all the source code +TABLE assemble HOST arzach + + CMD svnsync RUN "assemble_svnsync" + AFTER init/* + AFTER OPT irafx_update + diff --git a/sm/assemble_dev.sm b/sm/assemble_dev.sm new file mode 100644 index 0000000..21af101 --- /dev/null +++ b/sm/assemble_dev.sm @@ -0,0 +1,19 @@ +# arzach assembles all the source code +TABLE assemble HOST arzach + + CMD dev.stsci_python RUN "assemble_stsci_python dev" + AFTER init/* + AFTER svnsync + + CMD dev.stsci_iraf RUN "assemble_stsci_iraf dev" + AFTER init/* + AFTER svnsync + + CMD dev.axe RUN "assemble_axe dev" + AFTER init/* + AFTER svnsync + + CMD dev.hstcal RUN "assemble_hstcal dev" + AFTER init/* + AFTER svnsync + diff --git a/sm/assemble_x.sm b/sm/assemble_x.sm new file mode 100644 index 0000000..68089c7 --- /dev/null +++ b/sm/assemble_x.sm @@ -0,0 +1,22 @@ +# arzach assembles all the source code +TABLE assemble HOST arzach + + CMD x.stsci_python RUN "assemble_stsci_python x" + AFTER init/* + AFTER svnsync + + CMD x.stsci_iraf RUN "assemble_stsci_iraf x" + AFTER init/* + AFTER svnsync + + CMD x.axe RUN "assemble_axe x" + AFTER init/* + AFTER svnsync + + CMD x.hstcal RUN "assemble_hstcal x" + AFTER init/* + AFTER svnsync + + CMD x.irafx_age RUN "irafx_age.py" + AFTER init/* + diff --git a/sm/build_astrolib.sm b/sm/build_astrolib.sm new file mode 100644 index 0000000..fdeb5de --- /dev/null +++ b/sm/build_astrolib.sm @@ -0,0 +1,5 @@ + +TABLE build HOST arzach + CMD dev.astrolib RUN "build_astrolib" + AFTER arzach:assemble/dev.astrolib + diff --git a/sm/build_dev.sm b/sm/build_dev.sm new file mode 100644 index 0000000..7011d5a --- /dev/null +++ b/sm/build_dev.sm @@ -0,0 +1,75 @@ +## TODO: +## add builds on ssbwebv1 for those things that we actually care about + +#################### +#################### + +# install stsci_python into default environment +# build hstcal +# - everywhere +TABLE build HOST herbert thor arzach bond cadeau banana + CMD dev.py2.7 RUN "build_stsci_python dev 2.7" + AFTER init/* + AFTER *:assemble/dev.stsci_python + CMD dev.hstcal RUN "build_hstcal dev" + AFTER init/* + AFTER *:assemble/dev.hstcal + +# older python environments +# - arzach only +TABLE build HOST arzach + CMD dev.py2.6 RUN "build_stsci_python dev 2.6" + AFTER init/* + AFTER *:assemble/dev.stsci_python + + CMD dev.py2.5 RUN "build_stsci_python dev 2.5" + AFTER init/* + AFTER *:assemble/dev.stsci_python + +# stsdas and friends +# - 32 bit only +TABLE build HOST herbert bond + CMD dev.axe RUN "build_axe dev" + AFTER init/* + AFTER *:assemble/dev.axe + + CMD dev.stsci_iraf RUN "build_stsci_iraf dev" + AFTER init/* + AFTER *:assemble/dev.stsci_iraf + AFTER build/dev.axe + + CMD dev.stsci_iraf_log RUN "build_stsci_iraf_log dev" + AFTER init/* + AFTER build/dev.stsci_iraf + + CMD dev.stsci_iraf_help RUN "build_stsci_iraf_help dev" + AFTER init/* + AFTER build/dev.stsci_iraf + + +# stsdas for 64 bit machines - get it from a related 32 bit system +TABLE build HOST thor arzach + CMD dev.stsci_iraf_64hack RUN "build_stsci_iraf_64hack dev herbert" + AFTER herbert:build/dev.stsci_iraf* + +TABLE build HOST cadeau banana + CMD dev.stsci_iraf_64hack RUN "build_stsci_iraf_64hack dev bond" + AFTER bond:build/dev.stsci_iraf* + +# stsci_python documentation +# - one machine only +TABLE build HOST arzach + CMD dev.stsci_python_sphinxdocs RUN "build_sphinxdocs dev 2.7" + AFTER build/dev.py2.7 + +# old epydoc documentation - only works on thor; hope we can get rid of +# epydoc sooner than we have to do anything about this. +TABLE build HOST thor + CMD dev.stsci_python_epydoc RUN "/thor/data2/iraf/epydoc_test/nightly" + AFTER build/dev.py2.7 + +# stamp the IRAF banner file when the builds are complete +TABLE stamp HOST herbert thor arzach bond cadeau banana + CMD dev RUN "build_stamp dev" + AFTER build/* + diff --git a/sm/build_x.sm b/sm/build_x.sm new file mode 100644 index 0000000..43377f7 --- /dev/null +++ b/sm/build_x.sm @@ -0,0 +1,80 @@ +## TODO: +## add builds on ssbwebv1 for those things that we actually care about + +#################### +#################### + +# arzach assembles all the source code +TABLE assemble HOST arzach + + CMD svnsync RUN "assemble_svnsync" + AFTER init/* + AFTER OPT irafx_update + + CMD x.stsci_python RUN "assemble_stsci_python x" + AFTER init/* + AFTER svnsync + + CMD x.stsci_iraf RUN "assemble_stsci_iraf x" + AFTER init/* + AFTER svnsync + + CMD x.axe RUN "assemble_axe x" + AFTER init/* + AFTER svnsync + + CMD x.hstcal RUN "assemble_hstcal x" + AFTER init/* + AFTER svnsync + + +#################### +#################### + +# install stsci_python into default environment +# build hstcal +# - everywhere +TABLE build HOST herbert thor arzach bond cadeau banana + CMD x.py2.7 RUN "build_stsci_python x 2.7" + AFTER init/* + AFTER *:assemble/x.stsci_python + CMD x.hstcal RUN "build_hstcal x" + AFTER init/* + AFTER *:assemble/x.hstcal + +# stsdas and friends +# - 32 bit only +TABLE build HOST herbert bond + CMD x.axe RUN "build_axe x" + AFTER init/* + AFTER *:assemble/x.axe + + CMD x.stsci_iraf RUN "build_stsci_iraf x" + AFTER init/* + AFTER *:assemble/x.stsci_iraf + AFTER build/x.axe + + CMD x.stsci_iraf_log RUN "build_stsci_iraf_log x" + AFTER init/* + AFTER build/x.stsci_iraf + + CMD x.stsci_iraf_help RUN "build_stsci_iraf_help x" + AFTER init/* + AFTER build/x.stsci_iraf + + +# stsdas for 64 bit machines - get it from a related 32 bit system +TABLE build HOST thor arzach + CMD x.stsci_iraf_64hack RUN "build_stsci_iraf_64hack x herbert" + AFTER herbert:build/x.stsci_iraf* + +TABLE build HOST cadeau banana + CMD x.stsci_iraf_64hack RUN "build_stsci_iraf_64hack x bond" + AFTER bond:build/x.stsci_iraf* + + +# stamp the IRAF banner file when the builds are complete +TABLE stamp HOST herbert thor arzach bond cadeau banana + CMD x RUN "build_stamp x" + AFTER build/* + diff --git a/sm/cdbs.sm b/sm/cdbs.sm new file mode 100644 index 0000000..178ae0b --- /dev/null +++ b/sm/cdbs.sm @@ -0,0 +1,13 @@ +TABLE cdbs HOST bond cadeau banana + CMD local_copy RUN "cdbs_copy" + AFTER init/* + +TABLE cdbs HOST bond + CMD mac_package RUN "/Users/iraf/daily_build/mac_package/make_packages cdbs" + AFTER cdbs/local_copy + +TABLE cdbs HOST arzach + CMD ftp RUN "cdbs_arc" + AFTER init/* + + diff --git a/sm/dist_dev.sm b/sm/dist_dev.sm new file mode 100644 index 0000000..1fd933b --- /dev/null +++ b/sm/dist_dev.sm @@ -0,0 +1,106 @@ +#################### +#################### + +# regular distributions + +TABLE distribute HOST herbert thor arzach + CMD dev.iraf RUN "synctool - irafdev" + AFTER stamp/dev + CMD dev.pyssg RUN "synctool - pyssgdev" + AFTER stamp/dev + CMD dev.stsci_iraf RUN "synctool - stsci_iraf_dev" + AFTER stamp/dev + CMD dev.hstcal RUN "synctool - hstcaldev" + AFTER stamp/dev + CMD dev.motd RUN "synctool - irafdev/iraf/unix/hlib/motd" + AFTER distribute/dev.iraf + +TABLE distribute HOST bond cadeau banana + CMD irafdev.pkg RUN "cd $HOME/daily_build/mac_package; ./clean ; ./build dev " AFTER stamp/dev + CMD irafdev.dmg RUN "cd $HOME/daily_build/mac_package; ./distribute dev -" AFTER irafdev.pkg + +# wads of special cases + +# jwcalibdev has local disk - some day it may do its own builds +TABLE distribute HOST arzach + CMD jwcalibdev.iraf RUN "synctool jwcalibdev: irafdev" + AFTER stamp/dev + CMD jwcalibdev.pyssg RUN "synctool jwcalibdev: pyssgdev" + AFTER stamp/dev + CMD jwcalibdev.stsci_iraf RUN "synctool jwcalibdev: stsci_iraf_dev" + AFTER stamp/dev + CMD jwcalibdev.hstcal RUN "synctool jwcalibdev: hstcaldev" + AFTER stamp/dev + CMD jwcalibdev.motd RUN "synctool jwcalibdev: irafdev/iraf/unix/hlib/motd" + AFTER jwcalibdev.iraf + +# goods - has RHE 5 only now + +TABLE distribute_other HOST arzach + CMD goods.iraf RUN "synctool goods12: irafdev" + AFTER stamp/dev + CMD goods.pyssg RUN "synctool goods12: pyssgdev" + AFTER stamp/dev + CMD goods.stsci_iraf RUN "synctool goods12: stsci_iraf_dev" + AFTER stamp/dev + CMD goods.hstcal RUN "synctool goods12: hstcaldev" + AFTER stamp/dev + CMD goods.motd RUN "synctool goods12: irafdev/iraf/unix/hlib/motd" + AFTER goods.iraf + +# witserv1 - who are these guys? + +TABLE distribute_other HOST arzach + CMD witserv1.iraf RUN "synctool witserv1: irafdev" + AFTER stamp/dev + CMD witserv1.pyssg RUN "synctool witserv1: pyssgdev" + AFTER stamp/dev + CMD witserv1.stsci_iraf RUN "synctool witserv1: stsci_iraf_dev" + AFTER stamp/dev + CMD witserv1.hstcal RUN "synctool witserv1: hstcaldev" + AFTER stamp/dev + CMD witserv1.motd RUN "synctool witserv1: irafdev/iraf/unix/hlib/motd" + AFTER witserv1.iraf + +# dmsinsvm - have a pipeline and irafx/irafdev on the same machine for INS + +TABLE distribute_other HOST arzach + CMD dmsinsvm.iraf RUN "synctool dmsinsvm: irafdev" + AFTER stamp/dev + CMD dmsinsvm.pyssg RUN "synctool dmsinsvm: pyssgdev" + AFTER stamp/dev + CMD dmsinsvm.stsci_iraf RUN "synctool dmsinsvm: stsci_iraf_dev" + AFTER stamp/dev + CMD dmsinsvm.hstcal RUN "synctool dmsinsvm: hstcaldev" + AFTER stamp/dev + CMD dmsinsvm.motd RUN "synctool dmsinsvm: irafdev/iraf/unix/hlib/motd" + AFTER dmsinsvm.iraf + +# UDF - another funded project with their own machines + +TABLE distribute_other HOST thor + CMD udf1.iraf RUN "synctool udf1: irafdev" + AFTER stamp/dev + CMD udf1.pyssg RUN "synctool udf1: pyssgdev" + AFTER stamp/dev + CMD udf1.stsci_iraf RUN "synctool udf1: stsci_iraf_dev" + AFTER stamp/dev + CMD udf1.hstcal RUN "synctool udf1: hstcaldev" + AFTER stamp/dev + CMD udf1.motd RUN "synctool udf1: irafdev/iraf/unix/hlib/motd" + AFTER udf1.iraf + +# royal - a beowulf cluster + +TABLE distribute_other HOST thor + CMD royal.iraf RUN "synctool royal: irafdev" + AFTER stamp/dev + CMD royal.pyssg RUN "synctool royal: pyssgdev" + AFTER stamp/dev + CMD royal.stsci_iraf RUN "synctool royal: stsci_iraf_dev" + AFTER stamp/dev + CMD royal.hstcal RUN "synctool royal: hstcaldev" + AFTER stamp/dev + CMD royal.motd RUN "synctool royal: irafdev/iraf/unix/hlib/motd" + AFTER royal.iraf + diff --git a/sm/dist_x.sm b/sm/dist_x.sm new file mode 100644 index 0000000..c18755a --- /dev/null +++ b/sm/dist_x.sm @@ -0,0 +1,111 @@ + +# we can make everything after this + +TABLE x_distribute HOST herbert thor arzach bond cadeau banana + CMD gate RUN "exit 0" + AFTER init/* + AFTER stamp/* + +# regular distributions + +TABLE x_distribute HOST herbert thor arzach + CMD x.iraf RUN "synctool - irafx" + AFTER x_distribute/gate + CMD x.pyssg RUN "synctool - pyssgx" + AFTER x_distribute/gate + CMD x.stsci_iraf RUN "synctool - stsci_iraf_x" + AFTER x_distribute/gate + CMD x.hstcal RUN "synctool - hstcalx" + AFTER x_distribute/gate + CMD x.motd RUN "synctool - irafx/iraf/unix/hlib/motd" + AFTER x_distribute/x.iraf + +TABLE x_distribute HOST bond cadeau banana + CMD irafx.pkg RUN "cd $HOME/daily_build/mac_package; ./clean ; ./build x " AFTER x_distribute/gate + CMD irafx.dmg RUN "cd $HOME/daily_build/mac_package; ./distribute x -" AFTER irafx.pkg + +# wads of special cases + +# jwcalibdev has local disk - some day it may do its own builds +TABLE x_distribute HOST arzach + CMD x.jwcalibdev.iraf RUN "synctool jwcalibdev: irafx" + AFTER x_distribute/gate + CMD x.jwcalibdev.pyssg RUN "synctool jwcalibdev: pyssgx" + AFTER x_distribute/gate + CMD x.jwcalibdev.stsci_iraf RUN "synctool jwcalibdev: stsci_iraf_x" + AFTER x_distribute/gate + CMD x.jwcalibdev.hstcal RUN "synctool jwcalibdev: hstcalx" + AFTER x_distribute/gate + CMD x.jwcalibdev.motd RUN "synctool jwcalibdev: irafx/iraf/unix/hlib/motd" + AFTER x.jwcalibdev.iraf + +# goods - has RHE 5 only now + +TABLE x_distribute_other HOST arzach + CMD goods.iraf RUN "synctool goods12: irafx" + AFTER x_distribute/gate + CMD goods.pyssg RUN "synctool goods12: pyssgx" + AFTER x_distribute/gate + CMD goods.stsci_iraf RUN "synctool goods12: stsci_iraf_x" + AFTER x_distribute/gate + CMD goods.hstcal RUN "synctool goods12: hstcalx" + AFTER x_distribute/gate + CMD goods.motd RUN "synctool goods12: irafx/iraf/unix/hlib/motd" + AFTER goods.iraf + +# witserv1 - who are these guys? + +TABLE x_distribute_other HOST arzach + CMD witserv1.iraf RUN "synctool witserv1: irafx" + AFTER x_distribute/gate + CMD witserv1.pyssg RUN "synctool witserv1: pyssgx" + AFTER x_distribute/gate + CMD witserv1.stsci_iraf RUN "synctool witserv1: stsci_iraf_x" + AFTER x_distribute/gate + CMD witserv1.hstcal RUN "synctool witserv1: hstcalx" + AFTER x_distribute/gate + CMD witserv1.motd RUN "synctool witserv1: irafx/iraf/unix/hlib/motd" + AFTER witserv1.iraf + +# dmsinsvm - have a pipeline and irafx/irafdev on the same machine for INS + +TABLE x_distribute_other HOST arzach + CMD dmsinsvm.iraf RUN "synctool dmsinsvm: irafx" + AFTER x_distribute/gate + CMD dmsinsvm.pyssg RUN "synctool dmsinsvm: pyssgx" + AFTER x_distribute/gate + CMD dmsinsvm.stsci_iraf RUN "synctool dmsinsvm: stsci_iraf_x" + AFTER x_distribute/gate + CMD dmsinsvm.hstcal RUN "synctool dmsinsvm: hstcalx" + AFTER x_distribute/gate + CMD dmsinsvm.motd RUN "synctool dmsinsvm: irafx/iraf/unix/hlib/motd" + AFTER dmsinsvm.iraf + +# UDF - another funded project with their own machines + +TABLE x_distribute_other HOST thor + CMD udf1.iraf RUN "synctool udf1: irafx" + AFTER x_distribute/gate + CMD udf1.pyssg RUN "synctool udf1: pyssgx" + AFTER x_distribute/gate + CMD udf1.stsci_iraf RUN "synctool udf1: stsci_iraf_x" + AFTER x_distribute/gate + CMD udf1.hstcal RUN "synctool udf1: hstcalx" + AFTER x_distribute/gate + CMD udf1.motd RUN "synctool udf1: irafx/iraf/unix/hlib/motd" + AFTER udf1.iraf + +# royal - a beowulf cluster + +TABLE x_distribute_other HOST thor + CMD royal.iraf RUN "synctool royal: irafx" + AFTER x_distribute/gate + CMD royal.pyssg RUN "synctool royal: pyssgx" + AFTER x_distribute/gate + CMD royal.stsci_iraf RUN "synctool royal: stsci_iraf_x" + AFTER x_distribute/gate + CMD royal.hstcal RUN "synctool royal: hstcalx" + AFTER x_distribute/gate + CMD royal.motd RUN "synctool royal: irafx/iraf/unix/hlib/motd" + AFTER royal.iraf + diff --git a/sm/init.sm b/sm/init.sm new file mode 100644 index 0000000..b19731f --- /dev/null +++ b/sm/init.sm @@ -0,0 +1,5 @@ +TABLE init HOST banana bond cadeau arzach thor herbert ssb aten jwcalibdev etcbrady + CMD sendscripts LOCAL "/eng/ssb/auto/steuermann_scripts/init_sendscripts %(hostname)s %(workdir)s" + CMD sysstat RUN "sysstat" + AFTER sendscripts + diff --git a/sm/pyetc.sm b/sm/pyetc.sm new file mode 100644 index 0000000..d1b4419 --- /dev/null +++ b/sm/pyetc.sm @@ -0,0 +1,20 @@ +#################### +#################### + +TABLE pyetc HOST etcbrady + + CMD trunk RUN "/data1/nightly/trunk/test-ppsetc -d trunk " + AFTER init/* + + CMD refactored RUN "/data1/nightly/refactored/test-ppsetc -d refactored " + AFTER trunk + AFTER init/* + + CMD jwst RUN "/data1/nightly/jwpetc_dev/test-ppsetc -d" + AFTER refactored + AFTER init/* + + CMD rt RUN "/data1/nightly/rt/rt.csh 7" + AFTER jwst + AFTER init/* + diff --git a/sm/test_common.sm b/sm/test_common.sm new file mode 100644 index 0000000..50b04cf --- /dev/null +++ b/sm/test_common.sm @@ -0,0 +1,16 @@ +TABLE test_setup HOST herbert thor arzach bond cadeau banana + CMD okify RUN "test_okify" + AFTER distribute/* + +# each of test_run_* says AFTER test_setup/* + +TABLE test_final HOST ssb + CMD import RUN "test_import daily" + AFTER *:test_run_*/* + CMD notify RUN "test_notify daily" + AFTER import + CMD sqlite_watch RUN "test_sqlite_watch" + AFTER import + CMD mysql_watch RUN "test_mysql_watch" + AFTER import + diff --git a/sm/test_dev.sm b/sm/test_dev.sm new file mode 100644 index 0000000..7d2b7ea --- /dev/null +++ b/sm/test_dev.sm @@ -0,0 +1,23 @@ +#################### +#################### + +IMPORT "test_common.sm" + + +# This structure is meant to serialize the regtests so that nothing else +# is going on at the same time. The regtests can consume all the CPUs. +TABLE test_run_dev HOST herbert thor arzach bond cadeau banana + CMD test2.7 RUN "test_rt dev 2.7" + AFTER test_setup/okify + AFTER test_setup/* + +TABLE test_run_dev HOST arzach + CMD test2.6 RUN "test_rt dev 2.6" + AFTER test2.7 + CMD test2.5 RUN "test_rt dev 2.5" + AFTER test2.6 + +TABLE test_run_dev HOST thor + CMD contact RUN "test_contact" + AFTER test2.7 + diff --git a/sm/test_import.sm b/sm/test_import.sm new file mode 100644 index 0000000..6238aba --- /dev/null +++ b/sm/test_import.sm @@ -0,0 +1,12 @@ +#################### +#################### + +TABLE test_finale HOST ssb + CMD dev.import RUN "test_import daily" + AFTER *:test_run_*/* + CMD dev.notify RUN "test_notify daily" + AFTER dev.import + CMD dev.sqlite_watch RUN "test_sqlite_watch" + AFTER dev.import + CMD dev.mysql_watch RUN "test_mysql_watch" + AFTER dev.import diff --git a/sm/test_x.sm b/sm/test_x.sm new file mode 100644 index 0000000..be22da3 --- /dev/null +++ b/sm/test_x.sm @@ -0,0 +1,22 @@ +#################### +#################### +IMPORT "test_common.sm" + +# This structure is meant to serialize the regtests so that nothing else +# is going on at the same time. The regtests can consume all the CPUs. +TABLE test_run_x HOST herbert thor arzach bond cadeau banana + CMD test2.7 RUN "test_rt x 2.7" + AFTER test_setup/okify + AFTER test_setup/* + AFTER OPT test_run_dev/* + +TABLE test_run_x HOST arzach + CMD test2.6 RUN "test_rt x 2.6" + AFTER test2.7 + CMD test2.5 RUN "test_rt x 2.5" + AFTER test2.6 + +TABLE test_run_x HOST thor + CMD contact RUN "test_contact" + AFTER test2.7 + diff --git a/steuermann/hosts.ini b/steuermann/hosts.ini index abc114c..d750134 100644 --- a/steuermann/hosts.ini +++ b/steuermann/hosts.ini @@ -86,6 +86,13 @@ like=linux:csh workdir=/arzach/data1/iraf/steuermann maxproc=4 +[etcbrady] +hostname=etcbrady +like=linux:csh +workdir=/home/iraf/sm_work +; because each thing wants all the processors +maxproc=1 + [localhost] like=ssb diff --git a/steuermann/nodes.py b/steuermann/nodes.py index 476734a..3ad4639 100644 --- a/steuermann/nodes.py +++ b/steuermann/nodes.py @@ -11,6 +11,11 @@ import fnmatch class command_tree(object): + # list of additional files to import after this one; this is just + # here so we have a place to put it. It is appended by the parser + # and consumed by read_file_list() + import_list = [ ] + # a dict that maps currently known node names to node objects node_index = None @@ -315,11 +320,34 @@ current_file_name = None def read_file_list( file_list ) : global current_file_name di = command_tree( ) - for x in file_list : - current_file_name = x - sc = specfile.specfileScanner( open(x,'r').read() ) + imported = { } + while len(file_list) > 0 : + + print "START",file_list + # first name off the list + current_file_name = file_list[0] + file_list = file_list[1:] + print "LIST",file_list + + # see if it imported already + if current_file_name in imported : + print "SKIP",current_file_name + continue + imported[current_file_name] = 1 + + print "READING ",current_file_name + # read/parse + sc = specfile.specfileScanner( open(current_file_name,'r').read() ) p = specfile.specfile( scanner=sc, data=di ) + result = specfile.wrap_error_reporter( p, 'start' ) + + # if there were any import statements in the file, add those files to the list + file_list += di.import_list + di.import_list = [ ] + + print "END",file_list + di.finish() return di @@ -328,5 +356,3 @@ if __name__=='__main__': n = read_file_list( sys.argv[1:] ) print show_nodes(n.node_index) - - diff --git a/steuermann/run.py b/steuermann/run.py index 388b366..77db2f1 100644 --- a/steuermann/run.py +++ b/steuermann/run.py @@ -29,9 +29,14 @@ import os.path import traceback import sys import errno - import ConfigParser +def config_yes_no(d,which) : + if not which in d : + return False + s = d[which] + return s.strip()[0].lower() in ( 'y', 't', '1' ) + debug=0 ##### @@ -73,8 +78,15 @@ class runner(object): # start a process def run( self, node, run_name, logfile_name, no_run = False ): + '''run a process + return is: + D - host disabled + M - not run max proc limit + R - running +''' try : + try : args = self.get_host_info(node.host) except Exception, e : @@ -83,13 +95,16 @@ class runner(object): print e raise + if ( config_yes_no(args,'disable') ) : + return 'D' + hostname = args['hostname'] if 'maxproc' in args : n = int(self.howmany.get(hostname,0)) if n >= int(args['maxproc']) : # print "decline to run %s - %d other already running"%(node.name,n) - return False + return 'M' n = n + 1 self.howmany[hostname] = n @@ -154,8 +169,9 @@ class runner(object): logfile.flush() # debug - just say the name of the node we would run - if no_run : - run = [ 'echo', 'no_run - node=', node.name ] + + if ( no_run ) : + run = [ 'echo', 'disable run - node=', node.name ] # start running the process if debug : @@ -174,7 +190,7 @@ class runner(object): # remember the process is running self.all_procs[node.name] = n - return True + return 'R' except Exception, e : log_traceback() diff --git a/steuermann/run_all.py b/steuermann/run_all.py index 0c81fbf..b87ba7d 100644 --- a/steuermann/run_all.py +++ b/steuermann/run_all.py @@ -451,10 +451,21 @@ def run_step( runner, xnodes, run_name, db ) : no_sleep = 1 keep_running = 1 else : - if tmp : - # returns true/false whether it actually ran it - it may not because of resource limits + if tmp == 'R' : db.execute("UPDATE sm_status SET start_time = ?, status = 'R' WHERE ( run = ? AND host = ? AND tablename = ? AND cmd = ? )", ( str(datetime.datetime.now()), run_name, host, table, cmd ) ) + elif tmp == 'D' : + # same as skip above + x.finished = 1 + no_sleep = 1 + keep_running = 1 + db.execute("UPDATE sm_status SET start_time = ?, status = 'S' WHERE ( run = ? AND host = ? AND tablename = ? AND cmd = ? )", + ( str(datetime.datetime.now()), run_name, host, table, cmd ) ) + elif tmp == 'M' : + # hit max proc - not run, but try again later + pass + else : + print "WARNING: runner.run() returned unknown code %s"%str(tmp) db.commit() diff --git a/steuermann/run_cron.py b/steuermann/run_cron.py index 5779e1e..f2a01b3 100644 --- a/steuermann/run_cron.py +++ b/steuermann/run_cron.py @@ -58,18 +58,32 @@ def main() : node.script_type = 'r' # remote runner = steuermann.run.runner( nodes = { node.name : node } ) - runner.run( node=node, run_name='', logfile_name = steuermann.config.logdir + '/cron/' + logfile ) - - n = 0.1 - while 1 : - exited = runner.poll() - if exited : - break - if n < 2.0 : - n = n * 2.0 - time.sleep(n) - - status = exited[1] + logname = logfile_name = steuermann.config.logdir + '/cron/' + logfile + st = runner.run( node=node, run_name='', logfile_name = logname ) + + if st == 'D' : + fp = open(logname,"w") + fp.write('execution on host is disabled in hosts.ini\n') + fp.close() + status = 'S' + + elif st == 'M' : + fp = open(logname,"w") + fp.write('host is at max proc limit - how did this happen?') + fp.close() + status = '?' + + elif st == 'R' : + n = 0.1 + while 1 : + exited = runner.poll() + if exited : + break + if n < 2.0 : + n = n * 2.0 + time.sleep(n) + + status = exited[1] end_time = datetime.datetime.now() td = end_time - start_time diff --git a/steuermann/specfile.exy b/steuermann/specfile.exy index 7cdf1f3..cb2aa36 100644 --- a/steuermann/specfile.exy +++ b/steuermann/specfile.exy @@ -24,6 +24,7 @@ parser specfile: token string: '"[^"]*"' token SLASH: "/" token COLON: ":" + token IMPORT: "IMPORT" rule start: table_list END {{ return table_list }} @@ -38,6 +39,7 @@ parser specfile: command_list # command_list is a list of (command, pos) where command is the text from the file and pos is the location in the file {{ self.data.add_command_list( table_name, hostlist, command_list ) }} + | IMPORT string {{ self.data.import_list.append( string[1:-1] ) }} rule command_list: # one or more commands, appended together into a list @@ -34,7 +34,7 @@ TABLE assemble HOST arzach # install stsci_python into default environment # build hstcal # - everywhere -TABLE build HOST herbert thor arzach bond cadeau +TABLE build HOST herbert thor arzach bond cadeau banana CMD x.py2.7 RUN "build_stsci_python x 2.7" AFTER init/* AFTER *:assemble/x.stsci_python @@ -68,13 +68,13 @@ TABLE build HOST thor arzach CMD x.stsci_iraf_64hack RUN "build_stsci_iraf_64hack x herbert" AFTER herbert:build/x.stsci_iraf* -TABLE build HOST cadeau +TABLE build HOST cadeau banana CMD x.stsci_iraf_64hack RUN "build_stsci_iraf_64hack x bond" AFTER bond:build/x.stsci_iraf* # stamp the IRAF banner file when the builds are complete -TABLE stamp HOST herbert thor arzach bond cadeau +TABLE stamp HOST herbert thor arzach bond cadeau banana CMD x RUN "build_stamp x" AFTER build/* @@ -1,9 +1,10 @@ # we can make everything after this -TABLE x_distribute HOST herbert thor arzach bond cadeau +TABLE x_distribute HOST herbert thor arzach bond cadeau banana CMD gate RUN "exit 0" AFTER init/* + AFTER stamp/* # regular distributions @@ -19,7 +20,7 @@ TABLE x_distribute HOST herbert thor arzach CMD x.motd RUN "synctool - irafx/iraf/unix/hlib/motd" AFTER x_distribute/x.iraf -TABLE x_distribute HOST bond cadeau +TABLE x_distribute HOST bond cadeau banana CMD irafx.pkg RUN "cd $HOME/daily_build/mac_package; ./clean ; ./build x " AFTER x_distribute/gate CMD irafx.dmg RUN "cd $HOME/daily_build/mac_package; ./distribute x -" AFTER irafx.pkg |