diff options
| -rwxr-xr-x | bin/build.sh | 36 | ||||
| -rwxr-xr-x | bin/docs_jwst.sh | 5 | ||||
| -rwxr-xr-x | bin/tests_dev27.sh | 4 | ||||
| -rwxr-xr-x | bin/tests_jwst_dev27.sh | 4 | ||||
| -rwxr-xr-x | bin/tests_okify.sh | 6 | ||||
| -rwxr-xr-x | bin/tests_okify_tables.sh | 33 | ||||
| -rwxr-xr-x | bin/tests_public27.sh | 7 | ||||
| -rwxr-xr-x | bin/update_all.sh | 14 | ||||
| -rw-r--r-- | etc/manifest-dev.lst | 11 | ||||
| -rw-r--r-- | etc/manifest-oneoff.lst | 1 | ||||
| -rw-r--r-- | etc/manifest-public.lst | 4 | ||||
| -rw-r--r-- | etc/porcelain-extra | 1 | ||||
| -rw-r--r-- | include/conda_porcelain.sh | 40 | ||||
| -rw-r--r-- | include/error.sh | 136 | ||||
| -rwxr-xr-x | include/midnight_special.sh | 4 | ||||
| -rw-r--r-- | include/post-common.sh | 2 | ||||
| -rw-r--r-- | include/pre-common.sh | 2 | ||||
| -rwxr-xr-x | interactive.sh | 2 |
18 files changed, 278 insertions, 34 deletions
diff --git a/bin/build.sh b/bin/build.sh index 177c47c..0a60284 100755 --- a/bin/build.sh +++ b/bin/build.sh @@ -28,10 +28,6 @@ # exit regardless of exit method. Signals and general exit calls are ALL # handled equally. # -# If you do, you will quickly fill up the partition. Builds can be as large -# as 4GB, and potentially larger as time goes on. Don't be a doofus; always -# run "porcelain_deinit". You have been warned. -# # Never run a build outside of the "midnight_special" environment. The IRAF # account is a total mess. "midnight_special" performs a magical ritutal # that reassigns $HOME and effectively carpet bombs the original environment. @@ -40,8 +36,10 @@ source /eng/ssb/auto/astroconda/include/midnight_special.sh source /eng/ssb/auto/astroconda/include/sysinfo.sh -source /eng/ssb/auto/astroconda/include/conda_porcelain.sh +source /eng/ssb/auto/astroconda/include/error.sh source /eng/ssb/auto/astroconda/include/logger.sh +source /eng/ssb/auto/astroconda/include/conda_porcelain.sh + function warning_sleep { @@ -52,7 +50,7 @@ function warning_sleep echo "YOU PROBABLY DO NOT WANT THIS!" echo "Sleeping for $wtime second(s) just in case." - #sleep $wtime + sleep $wtime echo "Continuing..." } @@ -65,12 +63,16 @@ function repo_transfer if [[ -z $path ]]; then echo "transfer_repo requires a path." + error_set _E_FLAG_FAILED + error_set _E_FLAG_BAD_ARGUMENT exit 1 fi if [[ $path == *${repo_arch} ]]; then echo "tranfer_repo received an invalid path: $path" echo "(Remove the trailing /$repo_arch)" + error_set _E_FLAG_FAILED + error_set _E_FLAG_BAD_ARGUMENT exit 1 fi @@ -83,7 +85,8 @@ function repo_transfer retval=$? if [[ $retval > 0 ]]; then - exit $? + error_set _E_FLAG_BAD_RETVAL + return $? fi } @@ -92,11 +95,13 @@ function repo_index local path="$1" if [[ -z $path ]]; then echo "index_repo requires a path" + error_set _E_FLAG_BAD_ARGUMENT exit 1 fi if [[ ! -d $path ]]; then echo "$path does not exist." + error_set _E_FLAG_BAD_ARGUMENT exit 1 fi @@ -105,7 +110,8 @@ function repo_index retval=$? if [[ $retval > 0 ]]; then - exit $retval + error_set _E_FLAG_BAD_RETVAL + return $retval fi } @@ -317,6 +323,7 @@ pushd "$PORCELAIN_PREFIX" conda install --yes --quiet conda-build=1.18.1 conda=3.19.1 if [[ $? > 0 ]]; then echo "Unable to install conda-build, so stopping." + error_set _E_FLAG_HALT_AND_CATCH_FIRE exit 1 fi echo @@ -324,6 +331,7 @@ pushd "$PORCELAIN_PREFIX" git clone $repo_git if [[ $? > 0 ]]; then echo "Unable to clone recipe repository $repo_git, so stopping." + error_set _E_FLAG_HALT_AND_CATCH_FIRE exit 1 fi echo @@ -337,6 +345,10 @@ pushd "$PORCELAIN_PREFIX" fi logger $LOGDIR/${pkg}.log $build_command $pkg + + if [[ $? > 0 ]]; then + error_set _E_FLAG_BAD_PACKAGE + fi done else # Pretty much the worst thing you could ever WANT to do... @@ -344,17 +356,23 @@ pushd "$PORCELAIN_PREFIX" for pkg in * do [[ ! -f $pkg/meta.yaml ]] && continue + logger $LOGDIR/${pkg}.log $build_command $pkg + + if [[ $? > 0 ]]; then + error_set _E_FLAG_BAD_PACKAGE + fi done fi echo '----' logger repo_transfer.log repo_transfer "$repo_deposit" + echo '----' logger repo_index.log repo_index "$repo_deposit/$repo_arch" + echo '----' popd popd porcelain_deinit - diff --git a/bin/docs_jwst.sh b/bin/docs_jwst.sh index 1542cdd..1d01027 100755 --- a/bin/docs_jwst.sh +++ b/bin/docs_jwst.sh @@ -1,5 +1,6 @@ #!/bin/bash source /eng/ssb/auto/astroconda/include/midnight_special.sh +source /eng/ssb/auto/astroconda/include/error.sh source /eng/ssb/auto/astroconda/include/conda_porcelain.sh source /eng/ssb/auto/astroconda/include/texlive.sh @@ -20,8 +21,8 @@ pushd "$PORCELAIN_PREFIX" --yes \ --quiet \ --override-channels \ - -c defaults \ - -c $repo_conda sphinx=1.3.5 jwst stsci.sphinxext + -c $repo_conda \ + -c defaults sphinx=1.3.5 jwst stsci.sphinxext source activate $build_env diff --git a/bin/tests_dev27.sh b/bin/tests_dev27.sh index a1cd0b9..532a503 100755 --- a/bin/tests_dev27.sh +++ b/bin/tests_dev27.sh @@ -11,7 +11,7 @@ repo=http://ssb.stsci.edu/conda-dev source activate rt_${context}27 # Update environment -conda update -q -y --override-channels -c defaults -c $repo --all +conda update -q -y --override-channels -c $repo -c defaults --all source /eng/ssb/auto/astroconda/include/post-common.sh @@ -31,6 +31,8 @@ set -x [[ -d $LOGDIR ]] && [[ $LOGDIR != ^/$ ]] && rm -f "$LOGDIR/*" pushd $LOGDIR time pdkrun --parallel=${CPU_COUNT} -r "${tests[@]}" + retval=$? popd cat ${PDK_LOG}* | ssh iraf@ssb "irafdev ; pdk import -" +exit $retval diff --git a/bin/tests_jwst_dev27.sh b/bin/tests_jwst_dev27.sh index bb8878b..cf702ec 100755 --- a/bin/tests_jwst_dev27.sh +++ b/bin/tests_jwst_dev27.sh @@ -11,7 +11,7 @@ repo=http://ssb.stsci.edu/conda-dev source activate rt_${context}27 # Update environment -conda update -q -y --override-channels -c defaults -c $repo --all +conda update -q -y --override-channels -c $repo -c defaults --all source /eng/ssb/auto/astroconda/include/post-common.sh @@ -28,6 +28,8 @@ set -x [[ -d $LOGDIR ]] && [[ $LOGDIR != ^/$ ]] && rm -f "$LOGDIR/*" pushd $LOGDIR time pdkrun --parallel=${CPU_COUNT} -r "${tests[@]}" + retval=$? popd cat ${PDK_LOG}* | ssh iraf@ssb "irafdev ; pdk import -" +exit $retval diff --git a/bin/tests_okify.sh b/bin/tests_okify.sh index b46c43d..4b169fb 100755 --- a/bin/tests_okify.sh +++ b/bin/tests_okify.sh @@ -2,6 +2,8 @@ source /eng/ssb/auto/astroconda/include/midnight_special.sh source /eng/ssb/auto/astroconda/include/pre-common.sh +source activate pandokia + if [ "$groupdir" = "" ] then groupdir=/eng/ssb @@ -31,9 +33,13 @@ mv $h.ok $h.ok.process echo START +set -x + pdk ok -w $h.ok.process status=$? +set +x + echo END exit $status diff --git a/bin/tests_okify_tables.sh b/bin/tests_okify_tables.sh new file mode 100755 index 0000000..f4308fc --- /dev/null +++ b/bin/tests_okify_tables.sh @@ -0,0 +1,33 @@ +#!/bin/bash +source /eng/ssb/auto/astroconda/include/midnight_special.sh +source /eng/ssb/auto/astroconda/include/pre-common.sh +hostname=`hostname` + +case "$hostname" in + ssbwebv1*) + echo processing okify records in pandokia database + ;; + *) + echo $hostname cannot perform this action + exit 1 + ;; +esac + +# The only 'irafdev' gave us that we actually needed... +export PATH=/usr/stsci/pyssgdev/Python-2.7.1/bin:$PATH +export PYTHONPATH=/home/iraf/py/lib/python:/usr/stsci/pyssgdev/2.7.1.stsci_python:/usr/stsci/pyssgdev/2.7.1 + +# Pandokia specific (rather not clobber these together) +export N=82 +export PATH=/ssbwebv1/data2/pandokia/c$N/bin:$PATH +export PYTHONPATH=/ssbwebv1/data2/pandokia/c$N/lib/python/:$PYTHONPATH + +which pdk + +set -x + +pdk ok + +set +x + +exit $? diff --git a/bin/tests_public27.sh b/bin/tests_public27.sh index 1fa26b3..7f83419 100755 --- a/bin/tests_public27.sh +++ b/bin/tests_public27.sh @@ -7,11 +7,14 @@ name=conda context=public repo=http://ssb.stsci.edu/astroconda +# use RTX +test_from+='x' + # Activate environment source activate rt_${context}27 # Update environment -conda update -q -y --override-channels -c defaults -c $repo --all +conda update -q -y --override-channels -c $repo -c defaults --all source /eng/ssb/auto/astroconda/include/post-common.sh @@ -31,6 +34,8 @@ set -x [[ -d $LOGDIR ]] && [[ $LOGDIR != ^/$ ]] && rm -f "$LOGDIR/*" pushd $LOGDIR time pdkrun --parallel=${CPU_COUNT} -r "${tests[@]}" + retval=$? popd cat ${PDK_LOG}* | ssh iraf@ssb "irafdev ; pdk import -" +exit $retval diff --git a/bin/update_all.sh b/bin/update_all.sh index 11a5024..a89ef28 100755 --- a/bin/update_all.sh +++ b/bin/update_all.sh @@ -5,6 +5,8 @@ source /eng/ssb/auto/astroconda/include/pre-common.sh repo_base=http://ssb.stsci.edu contexts=( dev public ) versions=( 27 35 ) +tree_base=$(dirname $test_from) +trees=( rt rtx ) echo '----' echo 'Updating base installation:' @@ -37,15 +39,21 @@ do fi echo '----' echo "Updating $context from $repo:" - conda update -q -y --override-channels -c defaults -c $repo -n $environ --all + conda update -q -y --override-channels -c $repo -c defaults -n $environ --all echo '----' echo "Forcing pandokia to exist:" - conda install -q -y --override-channels -c defaults -c $repo -n $environ pandokia + conda install -q -y --override-channels -c $repo -c defaults -n $environ pandokia done done echo '----' echo 'Updating regression tests:' -svn_update `for d in /srv/rt/*; do [[ -d $d/.svn ]] && echo $d; done` +for tree in "${trees[@]}" +do + base="$tree_base/$tree" + if [[ -d $base/.svn ]]; then + svn_update $base + fi +done echo '----' diff --git a/etc/manifest-dev.lst b/etc/manifest-dev.lst index bbc4a50..b8842f1 100644 --- a/etc/manifest-dev.lst +++ b/etc/manifest-dev.lst @@ -1,8 +1,9 @@ +astropy stsci.tools stsci.imagestats -metapackages/stsci-data-analysis -metapackages/stsci-hst -metapackages/stsci +stsci-data-analysis +stsci-hst +stsci acstools appdirs @@ -18,7 +19,6 @@ costools crds cube-tools d2to1 -decorator drizzle drizzlepac ds9 @@ -38,10 +38,8 @@ pandokia photutils poppy purge_path -pydrizzle pyds9 pyfftw -pyneb pyqtgraph pyraf pyregion @@ -53,7 +51,6 @@ reftools relic selenium shunit2 -specview specviz sphere sphinx_rtd_theme diff --git a/etc/manifest-oneoff.lst b/etc/manifest-oneoff.lst new file mode 100644 index 0000000..d1ab94b --- /dev/null +++ b/etc/manifest-oneoff.lst @@ -0,0 +1 @@ +drizzlepac diff --git a/etc/manifest-public.lst b/etc/manifest-public.lst index 787e2d8..009a264 100644 --- a/etc/manifest-public.lst +++ b/etc/manifest-public.lst @@ -18,7 +18,6 @@ costools crds cube-tools d2to1 -decorator drizzle drizzlepac ds9 @@ -37,10 +36,8 @@ pandokia photutils poppy purge_path -pydrizzle pyds9 pyfftw -pyneb pyqtgraph pyraf pyregion @@ -52,7 +49,6 @@ reftools relic selenium shunit2 -specview specviz sphere sphinx_rtd_theme diff --git a/etc/porcelain-extra b/etc/porcelain-extra new file mode 100644 index 0000000..5664e30 --- /dev/null +++ b/etc/porcelain-extra @@ -0,0 +1 @@ +git diff --git a/include/conda_porcelain.sh b/include/conda_porcelain.sh index d4de1b7..f04ee50 100644 --- a/include/conda_porcelain.sh +++ b/include/conda_porcelain.sh @@ -2,6 +2,7 @@ source /eng/ssb/auto/astroconda/include/sysinfo.sh porcelain_continuum_url=https://repo.continuum.io/miniconda porcelain_continuum_script=Miniconda3-latest-${sysinfo_platform}-${sysinfo_arch}.sh +porcelain_extra_config=/eng/ssb/auto/astroconda/etc/porcelain-extra PORCELAIN_ALREADY_DEAD=0 PORCELAIN_SIGNALED=0 @@ -106,6 +107,27 @@ function porcelain_run_installer echo "Dying..." exit 1 fi + + # do extraneous package installation + porcelain_extra_install +} + +function porcelain_extra_install +{ + if [[ -f $porcelain_extra_config ]]; then + porcelain_verify + + while read pkg + do + if [[ $pkg == "" ]]; then + continue + elif [[ $pkg == "#"* ]]; then + continue + fi + + conda install -y -q $pkg + done < $porcelain_extra_config + fi } function porcelain_deinit @@ -129,14 +151,28 @@ function porcelain_deinit # thourough. function porcelain_signal { + # Obtain last return value retval=$? + + # If error.sh recorded errors; use the count instead + if [[ -n $_E_COUNT ]] && [[ $_E_COUNT > 0 ]]; then + retval=$_E_COUNT + fi + + # Already signaled, so die if [[ $PORCELAIN_SIGNALED != 0 ]]; then - exit $? + exit $retval + fi + + # If error.sh has been activated; display error report + if [[ -n $_E_FLAGS ]] && [[ $_E_FLAGS != 0 ]]; then + echo '----' + error_report fi export PORCELAIN_SIGNALED=1 porcelain_deinit - exit $? + exit $retval } diff --git a/include/error.sh b/include/error.sh new file mode 100644 index 0000000..dd0ddce --- /dev/null +++ b/include/error.sh @@ -0,0 +1,136 @@ +#!/bin/bash + +_E_WIDTH=15 +_E_COUNT=0 +_E_FLAGS=0 +_E_FLAGS_STR="" + +_E_FLAG_FAILURE=1 +_E_FLAG_EXIT_STANDARD=2 +_E_FLAG_EXIT_NONSTANDARD=4 +_E_FLAG_EXIT_SIGNALED=8 +_E_FLAG_EXIT_PEBKAC=16 +_E_FLAG_HALT_AND_CATCH_FIRE=32 +_E_FLAG_BAD_ARGUMENT=64 +_E_FLAG_BAD_RETVAL=128 +_E_FLAG_BAD_PACKAGE=256 +_E_FLAG_RES4=512 +_E_FLAG_RES5=1024 +_E_FLAG_RES6=2048 +_E_FLAG_RES7=4096 +_E_FLAG_RES8=8192 +_E_FLAG_RES9=16384 +_E_FLAG_INTERNAL=32768 +#_E_FLAG_UNUSED=65536 + +function error_mask_report +{ + local flags=$1 + + for (( i=${_E_WIDTH}; i>=0; i-- )) + do + (( mask = flags & ( 1 << i ) )) + case $mask in + $_E_FLAG_FAILURE) + echo "Script failed." + ;; + $_E_FLAG_EXIT_STANDARD) + echo "Normal exit." + ;; + $_E_FLAG_EXIT_NONSTANDARD) + echo "Non-standard exit." + ;; + $_E_FLAG_EXIT_SIGNALED) + echo "Recievied signal." + ;; + $_E_FLAG_EXIT_PEBCAC) + echo "Problem exists between keyboard and chair." + ;; + $_E_FLAG_HALT_AND_CATCH_FIRE) + echo "A fatal error occurred." + ;; + $_E_FLAG_BAD_ARGUMENT) + echo "Bad argument." + ;; + $_E_FLAG_BAD_RETVAL) + echo "An external program exited abnormally." + ;; + $_E_FLAG_BAD_PACKAGE) + echo "A package failed to build." + ;; + $_E_FLAG_RES4) + echo "Reserved" + ;; + $_E_FLAG_RES5) + echo "Reserved" + ;; + $_E_FLAG_RES6) + echo "Reserved" + ;; + $_E_FLAG_RES7) + echo "Reserved" + ;; + $_E_FLAG_RES8) + echo "Reserved" + ;; + $_E_FLAG_RES9) + echo "Reserved" + ;; + $_E_FLAG_INTERNAL) + echo "Error handler experienced an internal error." + ;; + *) + + ;; + esac + done +} + +function error_report +{ + local flags=`error_mask_string` + printf "Error count: %16s\n" $_E_COUNT + printf "Error mask: %16s\n" $_E_FLAGS + printf "Error flags: %16s\n" $flags + echo "Error message(s):" + while read line + do + echo "* $line" + done< <(error_mask_report $_E_FLAGS) +} + +function error_set +{ + local flag=$1 + if [[ -z $flag ]]; then + echo "error.sh: Unable to set error flag: '$flag'" + flag=$_E_FLAG_INTERNAL + fi + (( _E_FLAGS |= $flag )) + (( _E_COUNT++ )) + #echo "Error flag modified: $_E_FLAGS_STR" +} + +function error_mask_string +{ + declare -a -i bits + local flags=$_E_FLAGS + local output="" + + for i in $(seq 0 $_E_WIDTH) + do + (( bits[i] = 0 )) + (( bit = flags >> i )) + if (( flags & ( 1 << i ) )); then + (( bits[i] = 1 )) + fi + done + + for (( x=${_E_WIDTH}; x>=0; x-- )) + do + output+=$(( bits[x] )) + done + + echo $output +} + diff --git a/include/midnight_special.sh b/include/midnight_special.sh index 5781588..90c3638 100755 --- a/include/midnight_special.sh +++ b/include/midnight_special.sh @@ -4,7 +4,9 @@ source /eng/ssb/auto/astroconda/include/host_config.sh #[[ "$USER" != "" ]] && exec -c $0 -export sm_base="$PWD" +if [[ -n $sm_run ]]; then + export sm_base="$PWD" +fi unset $(/usr/bin/env \ | egrep '^(\w+)=(.*)$' \ diff --git a/include/post-common.sh b/include/post-common.sh index 0ea77b6..c9fb7e1 100644 --- a/include/post-common.sh +++ b/include/post-common.sh @@ -30,7 +30,7 @@ export PDK_CONTEXT=$context:${PYTHON_VERSION} export PDK_LOG="$LOGDIR/${HOSTNAME}-$PDK_TESTRUN" echo '----' -echo "Applying SHELL fix (thanks Continnum)..." +echo "Applying SHELL fix (thanks Continuum)..." if [[ $SHELL == bash ]]; then export SHELL=/bin/bash elif [[ $SHELL == zsh ]]; then diff --git a/include/pre-common.sh b/include/pre-common.sh index ac14f6f..9a9ef9b 100644 --- a/include/pre-common.sh +++ b/include/pre-common.sh @@ -64,7 +64,7 @@ function svn_update() pushd "$elem" &>/dev/null echo "Updating $elem ..." - svn up --non-interactive --accept theirs-conflict + svn up --non-interactive --depth infinity --accept theirs-conflict popd &>/dev/null done diff --git a/interactive.sh b/interactive.sh index 2ddaa3f..c7813db 100755 --- a/interactive.sh +++ b/interactive.sh @@ -1,4 +1,4 @@ #!/bin/bash source /eng/ssb/auto/astroconda/include/midnight_special.sh export TERM=xterm -PS1="[special] \u@\h:\w$ " bash +PS1="[special]\u@\h:\w$ " bash |