Initial commit

9 files changed, 684 insertions, 0 deletions

diff --git a/bin/build.sh b/bin/build.sh
new file mode 100755
index 0000000..177c47c
--- /dev/null
+++ b/bin/build.sh
@@ -0,0 +1,360 @@
+#!/bin/bash
+#
+# A steuermann compatible conda build script.
+#
+# This differs from the original build script in that it does not iterate over
+# many Python or Numpy versions in an attempt to be fully comprehensive.
+# Instead, since we're being "controlled", so we TELL steuermann what we want
+# the script to perform at a high level (not this script), rather than making
+# assumptions about what is needed/wanted at build-time.
+#
+# The build script uses "porcelain", and is not to be confused with "fragile".
+#
+# Conda uses lock files to determine whether its build (or installation)
+# system is currently in use. Traditionally this means only ONE build may be
+# active at a time. Porcelain installs its own miniconda into a completely
+# unique temporary directory so that eliminates the problem entirely. The
+# "conda build" subsystems never talk to one another while executing.
+
+# You may run multiple builds at the same time so long as you activate
+# porcelain [correctly] per-run.
+#
+# Porcelain is a write-once-remove-immediately, "WORI", enviroment. Yes,
+# the name does imply "worry". Be sure to either copy or move important data
+# out of $PORCELAIN_PREFIX or $TMPDIR prior to exiting the script or calling
+# "porcelain_deinit" directly.
+#
+# A signal handler is in place to run "porcelain_deinit" automatically on
+# exit regardless of exit method. Signals and general exit calls are ALL
+# handled equally.
+#
+# If you do, you will quickly fill up the partition. Builds can be as large
+# as 4GB, and potentially larger as time goes on. Don't be a doofus; always
+# run "porcelain_deinit". You have been warned.
+#
+# Never run a build outside of the "midnight_special" environment. The IRAF
+# account is a total mess. "midnight_special" performs a magical ritutal
+# that reassigns $HOME and effectively carpet bombs the original environment.
+# Please do not modify "midnight_special" for any reason.
+#
+
+source /eng/ssb/auto/astroconda/include/midnight_special.sh
+source /eng/ssb/auto/astroconda/include/sysinfo.sh
+source /eng/ssb/auto/astroconda/include/conda_porcelain.sh
+source /eng/ssb/auto/astroconda/include/logger.sh
+
+function warning_sleep
+{
+    wtime=$1
+    if [[ -z $wtime ]]; then
+        wtime=30
+    fi
+
+    echo "YOU PROBABLY DO NOT WANT THIS!"
+    echo "Sleeping for $wtime second(s) just in case."
+    #sleep $wtime
+    echo "Continuing..."
+}
+
+function repo_transfer
+{
+    porcelain_verify
+
+    local repo_local="$PORCELAIN_DESTDIR/conda-bld/$repo_arch"
+    local path="$1"
+
+    if [[ -z $path ]]; then
+        echo "transfer_repo requires a path."
+        exit 1
+    fi
+
+    if [[ $path == *${repo_arch} ]]; then
+        echo "tranfer_repo received an invalid path: $path"
+        echo "(Remove the trailing /$repo_arch)"
+        exit 1
+    fi
+
+    echo "Transfering local repository $repo_local to $path"
+    rsync -aHv \
+        --exclude='repodata*' \
+        --exclude='.index*' \
+        "$repo_local" \
+        "$path/"
+
+    retval=$?
+    if [[ $retval > 0 ]]; then
+        exit $?
+    fi
+}
+
+function repo_index
+{
+    local path="$1"
+    if [[ -z $path ]]; then
+        echo "index_repo requires a path"
+        exit 1
+    fi
+
+    if [[ ! -d $path ]]; then
+        echo "$path does not exist."
+        exit 1
+    fi
+
+    echo "Indexing remote repository: $path"
+    conda index $path
+
+    retval=$?
+    if [[ $retval > 0 ]]; then
+        exit $retval
+    fi
+}
+
+function usage
+{
+    printf "%s: [-h] [-bm] (-p|-d)\n" $(basename $0)
+    echo "
+    Optional:
+
+    --bootstrap         -B  Build regardless if package exists upstream
+                            (Dangerous)
+    --branch [branch]   -b  Desired git branch 
+                            (OMIT trailing os-arch; e.g. /linux-64)
+    --help              -h  This message
+    --manifest [file]   -m  Ordered build list
+
+    Required:
+
+    --public            -p  Use public (astroconda)
+    --dev               -d  Use dev    (astroconda-dev)
+    --deposit [path]    -D  Destination for completed packages
+    --numpy [version]   -N  NumPy linkage version
+    --python [version]  -P  Python linksage version
+    "
+}
+
+# 775 | 664
+umask 002
+
+build_bootstrap=0
+muarg_public=0
+muarg_dev=0
+
+
+if [[ $# < 1 ]]; then
+    usage
+    exit 1
+fi
+
+while [[ $# > 0 ]]
+do
+    key="$1"
+    case "$key" in
+        --help|-h)
+            usage
+            exit 0
+        ;;
+        --branch|-b)
+            repo_git_branch="$2"
+            if [[ $repo_git_branch == -* ]] || [[ -z $repo_git_branch ]]; then
+                echo "--branch/-b requires an argument."
+                exit 1
+            fi
+            shift
+        ;;
+        --deposit|-D)
+            repo_deposit="$2"
+            if [[ $repo_deposit == -* ]] || [[ -z $repo_deposit ]]; then
+                echo "--deposit requires an argument."
+                exit 1
+            fi
+            shift
+        ;;
+        --python|-P)
+            build_python="$2"
+            if [[ $build_python == -* ]] || [[ -z $build_python ]]; then
+                echo "--python requires an argument."
+                exit 1
+            fi
+            shift
+        ;;
+        --numpy|-N)
+            build_numpy="$2"
+            if [[ $build_numpy == -* ]] || [[ -z $build_numpy ]]; then
+                echo "--numpy requires an argument."
+                exit 1
+            fi
+            shift
+        ;;
+        --public|-p)
+            muarg_public=1
+            if [[ $muarg_dev != 0 ]]; then
+                echo "--public is mutually exclusive with --dev"
+                exit 1
+            fi
+            repo_git=https://github.com/astroconda/astroconda-contrib
+            repo_conda=http://ssb.stsci.edu/astroconda
+        ;;
+        --dev|-d)
+            muarg_dev=1
+            if [[ $muarg_public != 0 ]]; then
+                echo "--dev is mutually exclusive with --public"
+                exit 1
+            fi
+            repo_git=https://github.com/astroconda/astroconda-dev
+            repo_conda=http://ssb.stsci.edu/conda-dev
+        ;;
+        --manifest|-m)
+            build_manifest="$2"
+            if [[ ! -f $build_manifest ]]; then
+                echo "'$build_manifest' does not exist."
+                exit 1
+            fi
+            shift
+        ;;
+        --bootstrap|-B)
+            build_bootstrap=1
+        ;;
+        *)
+            usage
+            echo "Unknown argument: $1"
+            echo
+            exit 1
+        ;;
+    esac
+    shift
+done
+
+if [[ -z $repo_git ]]; then
+    echo "Missing argument: --public or --dev is required"
+    exit 1
+elif [[ -z $repo_deposit ]]; then
+    echo "--deposit is required"
+    exit 1
+elif [[ -z $build_python ]]; then
+    echo "--python is required"
+    exit 1
+elif [[ -z $build_numpy ]]; then
+    echo "--numpy is required"
+    exit 1
+fi
+
+if [[ -z $repo_git_branch ]]; then
+    repo_git_branch=master
+fi
+
+if [[ ! -d $repo_deposit ]]; then
+    mkdir -p $repo_deposit
+    retval=$?
+    if [[ $? > 0 ]]; then
+        echo "Unable to create $repo_deposit"
+        exit 1
+    fi
+fi
+
+# Set repository tail
+repo_arch=`conda_arch`
+
+echo repo_git_branch=$repo_git_branch
+echo repo_git=$repo_git
+echo repo_deposit=$repo_deposit
+echo repo_arch=`conda_arch`
+echo build_python=$build_python
+echo build_numpy=$build_numpy
+echo build_manifest=$build_manifest
+
+if [[ $build_bootstrap != 0 ]]; then
+    echo "BOOTSTRAP MODE ACTIVE (DANGEROUS)"
+    warning_sleep 90
+fi
+
+if [[ -z $build_manifest ]]; then
+    echo "No manifest defined; building alphabetically."
+    warning_sleep 90
+fi
+
+build_command="conda build \
+    --quiet \
+    --python=$build_python \
+    --numpy=$build_numpy \
+    --override-channels \
+    -c defaults"
+
+if [[ $build_bootstrap == 0 ]]; then
+    build_command="$build_command --skip-existing -c $repo_conda"
+else
+    echo "Bootstrap mode is active... FINAL WARNING!"
+    warning_sleep 10
+fi
+
+
+porcelain_init
+
+# Setup logging
+if [[ -n $sm_base ]]; then
+    sm_logs="$sm_base/$sm_run/$sm_node"
+fi
+
+if [[ -n $sm_logs ]]; then
+    if [[ ! -d $sm_logs ]]; then
+        echo "Initializing steuermann log directory: $sm_logs"
+        mkdir -p $sm_logs
+    fi
+fi
+
+# Take note that we are testing $sm_base, not $sm_logs.
+if [[ -n $sm_base ]]; then
+    LOGDIR="$sm_logs"
+else
+    LOGDIR="$PORCELAIN_PREFIX/logs"
+    mkdir -p "$LOGDIR"
+fi
+
+# Begin build processs
+pushd "$PORCELAIN_PREFIX"
+    porcelain_get_installer
+    porcelain_run_installer
+
+    conda install --yes --quiet conda-build=1.18.1 conda=3.19.1
+    if [[ $? > 0 ]]; then
+        echo "Unable to install conda-build, so stopping."
+        exit 1
+    fi
+    echo
+
+    git clone $repo_git
+    if [[ $? > 0 ]]; then
+        echo "Unable to clone recipe repository $repo_git, so stopping."
+        exit 1
+    fi
+    echo
+
+    pushd "$(basename $repo_git)"
+        if [[ -n $build_manifest ]]; then
+            for pkg in $(cat $build_manifest)
+            do
+                if [[ -z $pkg ]] || [[ $pkg == \#* ]]; then
+                    continue
+                fi
+
+                logger $LOGDIR/${pkg}.log $build_command $pkg
+            done
+        else
+            # Pretty much the worst thing you could ever WANT to do...
+            # this will build things you DO NOT WANT. Use a manifest!
+            for pkg in *
+            do
+                [[ ! -f $pkg/meta.yaml ]] && continue
+                logger $LOGDIR/${pkg}.log $build_command $pkg
+            done
+        fi
+
+        echo '----'
+        logger repo_transfer.log repo_transfer "$repo_deposit"
+        echo '----'
+        logger repo_index.log repo_index "$repo_deposit/$repo_arch"
+        echo '----'
+    popd
+popd
+
+porcelain_deinit
+
diff --git a/bin/build_hmm.sh b/bin/build_hmm.sh
new file mode 100755
index 0000000..a73b9ee
--- /dev/null
+++ b/bin/build_hmm.sh
@@ -0,0 +1,81 @@
+#!/bin/bash
+#
+# A steuermann compatible conda build script.
+#
+# This differs from the original build script in that it does not iterate over
+# many Python or Numpy versions in an attempt to be fully comprehensive.
+# Instead, since we're being "controlled", so we TELL steuermann what we want
+# the script to perform at a high level (not this script), rather than making
+# assumptions about what is needed/wanted at build-time.
+#
+# The build script uses "porcelain", and is not to be confused with "fragile".
+#
+# Conda uses lock files to determine whether its build (or installation)
+# system is currently in use. Traditionally this means only ONE build may be
+# active at a time. Porcelain installs its own miniconda into a completely
+# unique temporary directory so that eliminates the problem entirely. The
+# "conda build" subsystems never talk to one another while executing.
+
+# You may run multiple builds at the same time so long as you activate
+# porcelain [correctly] per-run.
+#
+# Porcelain is a write-once-remove-immediately, "WORI", enviroment. Yes,
+# the name does imply "worry". Be sure to either copy or move important data
+# out of $PORCELAIN_PREFIX or $TMPDIR prior to calling "porcelain_deinit".
+#
+# DO NOT FORGET TO RUN "porcelain_deinit".
+#
+# If you do, you will quickly fill up the partition. Builds can be as large
+# as 4GB, and potentially larger as time goes on. Don't be a doofus; always
+# run "porcelain_deinit". You have been warned.
+#
+# Never run a build outside of the "midnight_special" environment. The IRAF
+# account is a total mess. "midnight_special" performs a magical ritutal
+# that reassigns $HOME and effectively carpet bombs the original environment.
+# Please do not modify "midnight_special" for any reason.
+#
+
+echo "BEFORE..."
+printenv | sort
+echo
+
+source /eng/ssb/auto/astroconda/include/midnight_special.sh
+source /eng/ssb/auto/astroconda/include/sysinfo.sh
+source /eng/ssb/auto/astroconda/include/logger.sh
+source /eng/ssb/auto/astroconda/include/conda_porcelain.sh
+
+echo "AFTER..."
+printenv | sort
+echo
+
+porcelain_init
+
+# Setup logging
+if [[ -n $sm_base ]]; then
+    sm_logs="$sm_base/$sm_run/$sm_node"
+fi
+
+if [[ -n $sm_logs ]]; then
+    if [[ ! -d $sm_logs ]]; then
+        echo "Initializing steuermann log directory: $sm_logs"
+        mkdir -p $sm_logs
+    fi
+fi
+
+# Take note that we are testing $sm_base, not $sm_logs.
+if [[ -n $sm_base ]]; then
+    LOGDIR="$sm_logs"
+else
+    LOGDIR="$PORCELAIN_PREFIX/logs"
+    mkdir -p "$LOGDIR"
+fi
+# Begin build processs
+pushd "$PORCELAIN_PREFIX"
+    porcelain_get_installer
+    porcelain_run_installer
+
+    logger $LOGDIR/a_test_file.log echo this output should be logged
+popd
+
+porcelain_deinit
+
diff --git a/bin/docs_jwst.sh b/bin/docs_jwst.sh
new file mode 100755
index 0000000..1542cdd
--- /dev/null
+++ b/bin/docs_jwst.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+source /eng/ssb/auto/astroconda/include/midnight_special.sh
+source /eng/ssb/auto/astroconda/include/conda_porcelain.sh
+source /eng/ssb/auto/astroconda/include/texlive.sh
+
+porcelain_init
+
+pushd "$PORCELAIN_PREFIX"
+    porcelain_get_installer
+    porcelain_run_installer
+    
+    build_env="$(basename $PORCELAIN_PREFIX)"
+    depot=/eng/ssb/websites/ssbpublic/doc/jwst_git
+    
+    repo_conda=http://ssb.stsci.edu/conda-dev
+    repo_git=https://github.com/stsci-jwst/jwst
+    repo_git_branch=master
+    
+    conda create -n "$build_env" \
+        --yes \
+        --quiet \
+        --override-channels \
+        -c defaults \
+        -c $repo_conda sphinx=1.3.5 jwst stsci.sphinxext
+
+    source activate $build_env
+
+    git clone "$repo_git"
+    pushd "$(basename $repo_git)"
+        git checkout $repo_git_branch
+        docs/mkdocs.sh -o "$depot"
+    popd
+popd
+
+porcelain_deinit
+
diff --git a/bin/prerelease.sh b/bin/prerelease.sh
new file mode 100755
index 0000000..f616e8d
--- /dev/null
+++ b/bin/prerelease.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+SRC=/eng/ssb/websites/ssbpublic/conda-dev
+DST=/eng/ssb/websites/ssbpublic/astroconda-prerelease/
+
+( cd $SRC \
+    && rsync -aH --relative */*jwst* $DST \
+    && conda index "$DST/linux-64" \
+    && conda index "$DST/osx-64" \
+    && cbc_repo2html "$DST/linux-64/repodata.json" > "$DST/linux-64/index.html" \
+    && cbc_repo2html "$DST/osx-64/repodata.json" > "$DST/osx-64/index.html"
+)
diff --git a/bin/tests_dev27.sh b/bin/tests_dev27.sh
new file mode 100755
index 0000000..a1cd0b9
--- /dev/null
+++ b/bin/tests_dev27.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+source /eng/ssb/auto/astroconda/include/midnight_special.sh
+source /eng/ssb/auto/astroconda/include/pre-common.sh
+
+# Set context (used here and in post-common.sh)
+name=conda
+context=dev
+repo=http://ssb.stsci.edu/conda-dev
+
+# Activate environment
+source activate rt_${context}27
+
+# Update environment
+conda update -q -y --override-channels -c defaults -c $repo --all
+
+source /eng/ssb/auto/astroconda/include/post-common.sh
+
+# Assign tests to run
+tests=(
+    $test_from/astrolib
+    $test_from/stsci_python
+    $test_from/betadrizzle
+    $test_from/axe
+    $test_from/hstcal
+    $test_from/stsdas
+)
+
+# Nuke existing logs, run the tests, import the tests
+set -x
+
+[[ -d $LOGDIR ]] && [[ $LOGDIR != ^/$ ]] && rm -f "$LOGDIR/*"
+pushd $LOGDIR
+    time pdkrun --parallel=${CPU_COUNT} -r "${tests[@]}"
+popd
+cat ${PDK_LOG}* | ssh iraf@ssb "irafdev ; pdk import -"
+
diff --git a/bin/tests_jwst_dev27.sh b/bin/tests_jwst_dev27.sh
new file mode 100755
index 0000000..bb8878b
--- /dev/null
+++ b/bin/tests_jwst_dev27.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+source /eng/ssb/auto/astroconda/include/midnight_special.sh
+source /eng/ssb/auto/astroconda/include/pre-common.sh
+
+# Set context (used here and in post-common.sh)
+name=conda
+context=dev
+repo=http://ssb.stsci.edu/conda-dev
+
+# Activate environment
+source activate rt_${context}27
+
+# Update environment
+conda update -q -y --override-channels -c defaults -c $repo --all
+
+source /eng/ssb/auto/astroconda/include/post-common.sh
+
+# Assign tests to run
+tests=(
+    $test_from/general
+    $test_from/more_tests
+    $test_from/test_functions
+)
+
+# Nuke existing logs, run the tests, import the tests
+set -x
+
+[[ -d $LOGDIR ]] && [[ $LOGDIR != ^/$ ]] && rm -f "$LOGDIR/*"
+pushd $LOGDIR
+    time pdkrun --parallel=${CPU_COUNT} -r "${tests[@]}"
+popd
+cat ${PDK_LOG}* | ssh iraf@ssb "irafdev ; pdk import -"
+
diff --git a/bin/tests_okify.sh b/bin/tests_okify.sh
new file mode 100755
index 0000000..b46c43d
--- /dev/null
+++ b/bin/tests_okify.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+source /eng/ssb/auto/astroconda/include/midnight_special.sh
+source /eng/ssb/auto/astroconda/include/pre-common.sh
+
+if [ "$groupdir" = "" ]
+then
+    groupdir=/eng/ssb
+else
+    echo groupdir is:
+    echo $groupdir
+fi
+
+
+# find our host name because it is used in the name of the okify file
+h=`hostname -s`
+echo $h
+
+# where the okify files are
+cd "$groupdir/tests/pdk_updates"
+
+ls -l
+
+if [[ ! -f $h.ok ]]; then
+    echo no $h.ok
+    exit 0
+fi
+
+# not processing directly from the active file
+rm -f $h.ok.process
+mv $h.ok $h.ok.process
+
+echo START
+
+pdk ok -w $h.ok.process
+status=$?
+
+echo END
+
+exit $status
+
diff --git a/bin/tests_public27.sh b/bin/tests_public27.sh
new file mode 100755
index 0000000..1fa26b3
--- /dev/null
+++ b/bin/tests_public27.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+source /eng/ssb/auto/astroconda/include/midnight_special.sh
+source /eng/ssb/auto/astroconda/include/pre-common.sh
+
+# Set context (used here and in post-common.sh)
+name=conda
+context=public
+repo=http://ssb.stsci.edu/astroconda
+
+# Activate environment
+source activate rt_${context}27
+
+# Update environment
+conda update -q -y --override-channels -c defaults -c $repo --all
+
+source /eng/ssb/auto/astroconda/include/post-common.sh
+
+# Assign tests to run
+tests=(
+    $test_from/astrolib
+    $test_from/stsci_python
+    $test_from/betadrizzle
+    $test_from/axe
+    $test_from/hstcal
+    $test_from/stsdas
+)
+
+# Nuke existing logs, run the tests, import the tests
+set -x
+
+[[ -d $LOGDIR ]] && [[ $LOGDIR != ^/$ ]] && rm -f "$LOGDIR/*"
+pushd $LOGDIR
+    time pdkrun --parallel=${CPU_COUNT} -r "${tests[@]}"
+popd
+cat ${PDK_LOG}* | ssh iraf@ssb "irafdev ; pdk import -"
+
diff --git a/bin/update_all.sh b/bin/update_all.sh
new file mode 100755
index 0000000..11a5024
--- /dev/null
+++ b/bin/update_all.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+source /eng/ssb/auto/astroconda/include/midnight_special.sh
+source /eng/ssb/auto/astroconda/include/pre-common.sh
+
+repo_base=http://ssb.stsci.edu
+contexts=( dev public )
+versions=( 27 35 )
+
+echo '----'
+echo 'Updating base installation:'
+conda update -q -y --all
+
+echo '----'
+for context in "${contexts[@]}"
+do
+    for version in "${versions[@]}"
+    do
+        case "$context" in
+            dev)
+                repo="$repo_base/conda-dev"
+            ;;
+            public)
+                repo="$repo_base/astroconda"
+            ;;
+            *)
+                echo "No repository available for: $context"
+                exit 1
+            ;;
+        esac
+
+        environ="rt_${context}${version}"
+	if [[ ! -d ~/miniconda3/envs/$environ ]]; then
+            echo '!!!!'
+            echo "No Conda environment for: $environ"
+            echo "Skipping..."
+            continue
+        fi
+        echo '----'
+        echo "Updating $context from $repo:"
+        conda update -q -y --override-channels -c defaults -c $repo -n $environ --all
+        echo '----'
+        echo "Forcing pandokia to exist:"
+        conda install -q -y --override-channels -c defaults -c $repo -n $environ pandokia
+    done
+done
+
+echo '----'
+echo 'Updating regression tests:'
+svn_update `for d in /srv/rt/*; do [[ -d $d/.svn ]] && echo $d; done`
+echo '----'
+