diff options
Diffstat (limited to 'include')
| -rw-r--r-- | include/conda_porcelain.sh | 142 | ||||
| -rw-r--r-- | include/host_config.sh | 13 | ||||
| -rw-r--r-- | include/logger.sh | 22 | ||||
| -rwxr-xr-x | include/midnight_special.sh | 18 | ||||
| -rw-r--r-- | include/post-common.sh | 45 | ||||
| -rw-r--r-- | include/pre-common.sh | 71 | ||||
| -rw-r--r-- | include/sysinfo.sh | 70 | ||||
| -rw-r--r-- | include/texlive.sh | 21 |
8 files changed, 402 insertions, 0 deletions
diff --git a/include/conda_porcelain.sh b/include/conda_porcelain.sh new file mode 100644 index 0000000..d4de1b7 --- /dev/null +++ b/include/conda_porcelain.sh @@ -0,0 +1,142 @@ +#!/bin/sh +source /eng/ssb/auto/astroconda/include/sysinfo.sh +porcelain_continuum_url=https://repo.continuum.io/miniconda +porcelain_continuum_script=Miniconda3-latest-${sysinfo_platform}-${sysinfo_arch}.sh +PORCELAIN_ALREADY_DEAD=0 +PORCELAIN_SIGNALED=0 + +function porcelain_init +{ + echo "Porcelain initializing..." + if [[ -z $HOME ]]; then + echo "\$HOME is not set, dying." + exit 1 + fi + + export TMPDIR="$HOME/bldtmp" + + if [[ ! -d $TMPDIR ]]; then + mkdir -p $TMPDIR + if [[ $? != 0 ]]; then + echo "Cannot create temporary storage directory '$TMPDIR', dying." + exit 1 + fi + fi + + export PORCELAIN_PREFIX=`mktemp -d -t porcelain.XXXXXXXXXX` + export PORCELAIN_TMPDIR="$PORCELAIN_PREFIX/tmp" + export PORCELAIN_DESTDIR="$PORCELAIN_PREFIX/porcelain" + export PATH="$PORCELAIN_DESTDIR/bin:$PATH" + echo "Prepended $PORCELAIN_DESTDIR/bin to PATH..." + + if [[ ! -d $PORCELAIN_TMPDIR ]]; then + mkdir -p "$PORCELAIN_TMPDIR" + fi + + export TMPDIR="$PORCELAIN_TMPDIR" + echo "Redirected TMPDIR to $PORCELAIN_TMPDIR" + echo "(Always use \$TMPDIR instead of /tmp for destructible storage)" + + echo "Activating signal handler... (will deinit on exit)" + trap porcelain_signal SIGINT SIGKILL SIGTERM EXIT +} + +function porcelain_verify +{ + local override="$1" + /bin/echo -n "Running safety check... " + + if [[ $override == '-f' ]] || [[ $override == '--force' ]]; then + echo "done (forced)" + return 0 + fi + + if [[ -z $TMPDIR ]]; then + echo "failed" + echo "TMPDIR='$TMPDIR'; we have lost trust in our environment. Dying." + exit 1 + fi + + if [[ ! -d $PORCELAIN_PREFIX ]]; then + echo "failed" + echo "PORCELAIN_PREFIX='$PORCELAIN_PREFIX'; mktemp must have failed. Dying." + exit 1 + fi + + # Note: we don't care about PORCELAIN_DESTDIR + echo "done" +} + +function porcelain_get_installer +{ + porcelain_verify + + retries=3 + wtime=5 + success=0 + count=0 + + echo "Obtaining $porcelain_continuum_script..." + + while [[ $count != $retries ]] + do + $sysinfo_fetch $sysinfo_fetch_args "$porcelain_continuum_url/$porcelain_continuum_script" + if [[ $? == 0 ]]; then + success=1 + break + fi + + echo "Retrying in $wtime second(s)..." + sleep $wtime + count=$(( count + 1 )) + done + + if [[ $success != 1 ]]; then + echo "Dying..." + exit 1 + fi +} + +function porcelain_run_installer +{ + porcelain_verify + + bash $porcelain_continuum_script -b -p $PORCELAIN_DESTDIR + if [[ $? > 0 ]]; then + echo "Dying..." + exit 1 + fi +} + +function porcelain_deinit +{ + if [[ $PORCELAIN_ALREADY_DEAD != 0 ]]; then + return + fi + + porcelain_verify + + echo "Deinitializing porcelain..." + if [[ -d $PORCELAIN_PREFIX ]]; then + echo "Removing $PORCELAIN_PREFIX" + rm -rf "$PORCELAIN_PREFIX" + fi + export PORCELAIN_ALREADY_DEAD=1 +} + +# Given the chance this script could be killed in a number of different +# ways, resulting in files left behind... we need to be pretty damn +# thourough. +function porcelain_signal +{ + retval=$? + if [[ $PORCELAIN_SIGNALED != 0 ]]; then + exit $? + fi + + export PORCELAIN_SIGNALED=1 + porcelain_deinit + + exit $? +} + diff --git a/include/host_config.sh b/include/host_config.sh new file mode 100644 index 0000000..a97778d --- /dev/null +++ b/include/host_config.sh @@ -0,0 +1,13 @@ +#(DEFAULT) +NEW_HOME=/srv/iraf + +#JWST +if [[ $HOSTNAME == *jwcalibdev* ]]; then + NEW_HOME=/data4/iraf_conda +fi + +#OSX +if [[ `uname -s` == Darwin ]]; then + NEW_HOME=/Users/shared/iraf_conda +fi + diff --git a/include/logger.sh b/include/logger.sh new file mode 100644 index 0000000..b8806e1 --- /dev/null +++ b/include/logger.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +function logger +{ + local logfile="$1" + if [[ $logfile != *.log ]]; then + echo "logger: log file is missing .log prefix, '$logfile'" + exit 1 + fi + + shift + + # Bash magic: return this exit value for the first pipe command + echo "Writing log: $logfile" + set -o pipefail + "$@" 2>&1 | tee $logfile + retval=$? + set +o pipefail + + return $retval +} + diff --git a/include/midnight_special.sh b/include/midnight_special.sh new file mode 100755 index 0000000..5781588 --- /dev/null +++ b/include/midnight_special.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Yeah, we're not dealing with the existing environment. Too many things can go wrong here. +# DROP EVERYTHING and start fresh in a different home directory +source /eng/ssb/auto/astroconda/include/host_config.sh + +#[[ "$USER" != "" ]] && exec -c $0 +export sm_base="$PWD" + +unset $(/usr/bin/env \ + | egrep '^(\w+)=(.*)$' \ + | egrep -vw 'SHLVL|USER|LANG|sm_run|sm_node|sm_logs|sm_base|node_dir|workdir|hostname' \ + | /usr/bin/cut -d= -f1) +export PATH=/sbin:/usr/sbin:/bin:/usr/bin:/usr/local/bin:/usr/local/sbin +export HOME=$NEW_HOME +export TERM=xterm +source /etc/profile +source /etc/bashrc +#printenv diff --git a/include/post-common.sh b/include/post-common.sh new file mode 100644 index 0000000..0ea77b6 --- /dev/null +++ b/include/post-common.sh @@ -0,0 +1,45 @@ +if [[ -z $context ]]; then + context=unknown +fi + +if [[ -z $name ]]; then + name=unknown +fi + +if [[ -z $test_to ]]; then + echo "\$test_to undefined. I refuse to continue. Did you forget to include pre-common?" + exit 1 +fi + +PYTHON_VERSION=$(python --version 2>&1 | awk '{ print $2 }') +CPU_COUNT=`python -c 'import multiprocessing as mp; print(mp.cpu_count()-1)'` + +DATETIME=$sm_run +if [[ -z $DATETIME ]]; then + echo "sm_run was undefined!" + DATETIME=broken_time_`date '+%Y-%m-%d-%H-%M-%s'` +fi + +export LOGDIR="$test_to/$context" +mkdir -p "$LOGDIR" + +if [[ -z $PDK_TESTRUN ]]; then + export PDK_TESTRUN=${name}-${DATETIME} +fi +export PDK_CONTEXT=$context:${PYTHON_VERSION} +export PDK_LOG="$LOGDIR/${HOSTNAME}-$PDK_TESTRUN" + +echo '----' +echo "Applying SHELL fix (thanks Continnum)..." +if [[ $SHELL == bash ]]; then + export SHELL=/bin/bash +elif [[ $SHELL == zsh ]]; then + export SHELL=/bin/zsh +else + # Don't ask why... + export SHELL=/bin/bash +fi + +echo '----' +printenv + diff --git a/include/pre-common.sh b/include/pre-common.sh new file mode 100644 index 0000000..ac14f6f --- /dev/null +++ b/include/pre-common.sh @@ -0,0 +1,71 @@ +function cleanup +{ + # Not a fan of ultra-verbose. This message will get lost. + set +x + echo "Trapped common exit signal (SIGINT | SIGTERM)" + /bin/echo + echo "TESTS WILL NOT BE IMPORTED" + /bin/echo + echo "Exiting..." + exit 1 +} +trap cleanup SIGINT SIGTERM + +test_from=/srv/rt +test_to=~/local/pillowfort + +if [[ `uname -s` == Darwin ]]; then + test_from=/Users/iraf/rt +fi + +if [[ $HOSTNAME == *jwcalibdev* ]]; then + test_from=/data1/jwst_rt +fi + +if [[ ! -d $test_to ]]; then + mkdir -p "$test_to" +fi + +export PATH=~/miniconda3/bin:$PATH +#export PATH=$PATH:~/local/fakedokia/bin +#export PYTHONPATH=~/local/fakedokia/lib +export CDBS=/grp/hst/cdbs/ +export crrefer=$CDBS +export PYSYN_CDBS=$CDBS + +function mkrt_paths +{ + args=("$@") + paths="" + + for elem in "${args[@]}" + do + path="$test_from/$elem" + if [[ ! -d $path ]]; then + echo "Warning: $path does not exist. Omitting." + continue + fi + paths+=($path) + done + printf $paths | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' +} + +function svn_update() +{ + args=("$@") + paths="" + + for elem in "${args[@]}" + do + if [[ ! -d $elem ]]; then + echo "Warning: $elem does not exist. Omitting." + continue + fi + + pushd "$elem" &>/dev/null + echo "Updating $elem ..." + svn up --non-interactive --accept theirs-conflict + popd &>/dev/null + + done +} diff --git a/include/sysinfo.sh b/include/sysinfo.sh new file mode 100644 index 0000000..76dbca8 --- /dev/null +++ b/include/sysinfo.sh @@ -0,0 +1,70 @@ +function check_which +{ + path="$1" + which $path &>/dev/null + status=$? + echo "$status" +} + +# For conda INSTALLERS +sysinfo_platform=`uname` +case $sysinfo_platform in + Darwin) + sysinfo_platform="MacOSX" + ;; + *) ;; +esac + +sysinfo_arch=`uname -m` +case $sysinfo_arch in + i*86) + sysinfo_arch="x86" + ;; + *) ;; +esac + +case $sysinfo_platform in + Linux) + sysinfo_fetch="wget" + sysinfo_fetch_args="-q" + ;; + *) + sysinfo_fetch="curl" + sysinfo_fetch_args="-s -L -O" + ;; +esac + +# For conda BUILDS +function conda_arch +{ + local platform= + local arch= + + case `uname` in + Linux) + platform="linux" + ;; + Darwin) + platform="osx" + ;; + *) + echo "Unsupported platform." + exit 1 + ;; + esac + + case `uname -m` in + i*86) + arch=32 + ;; + x86_64) + arch=64 + ;; + *) + echo "Unsupported architecture." + exit 1 + ;; + esac + + echo "${platform}-${arch}" +} diff --git a/include/texlive.sh b/include/texlive.sh new file mode 100644 index 0000000..a13b2d0 --- /dev/null +++ b/include/texlive.sh @@ -0,0 +1,21 @@ +#!/bin/bash +source /eng/ssb/auto/astroconda/include/sysinfo.sh + +TL_HOME=/eng/ssb/sw/texlive + +# Be realistic - I already know what's in there... 32/64-linux +if [[ $sysinfo_platform != Linux ]]; then + echo "FATAL: $TL_HOME does not contain libraries for your platform." + exit 1 +fi + +TL_PLATFORM=`$TL_HOME/install-tl --print-platform` +export PATH="$TL_HOME/bin/${TL_PLATFORM}:$PATH" + +if [[ `which latex` != $TL_HOME/* ]]; then + echo "WARNING: TexLive is not where we wanted it to be! $TL_HOME may be broken or missing." +else + echo "Using TexLive: $TL_HOME" +fi + + |