From c8ad645d8827bdd9fedc19a8c67f22aa016f8290 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Fri, 15 Feb 2019 22:11:56 -0500 Subject: Refactor build/publish --- Dockerfile | 5 +- build.sh | 80 ++++++++++++++++++++---- etc/jwstdp-latest/pip/001-setuptools | 2 - etc/jwstdp-latest/pip/002-testing | 6 -- etc/jwstdp-latest/pip/003-prerequisites | 4 -- etc/jwstdp-latest/pip/004-pipeline | 17 ----- etc/jwstdp-latest/pip/005-suppl | 1 - etc/jwstdp-latest/pkgs/000-jwst-depends.sh | 6 -- etc/jwstdp-latest/pkgs/001-cfitsio.sh | 15 ----- etc/jwstdp-latest/pkgs/002-fitsverify.sh | 16 ----- etc/jwstdp-latest/tasks/001-packages.sh | 38 ----------- etc/jwstdp-latest/tasks/002-python-packages.sh | 40 ------------ etc/jwstdp-latest/tasks/999-clean.sh | 18 ------ etc/jwstdp-snapshot/pip/001-setuptools | 2 + etc/jwstdp-snapshot/pip/002-testing | 6 ++ etc/jwstdp-snapshot/pip/003-prerequisites | 4 ++ etc/jwstdp-snapshot/pip/004-pipeline | 17 +++++ etc/jwstdp-snapshot/pip/005-suppl | 1 + etc/jwstdp-snapshot/pkgs/000-jwst-depends.sh | 6 ++ etc/jwstdp-snapshot/pkgs/001-cfitsio.sh | 15 +++++ etc/jwstdp-snapshot/pkgs/002-fitsverify.sh | 16 +++++ etc/jwstdp-snapshot/tasks/001-packages.sh | 38 +++++++++++ etc/jwstdp-snapshot/tasks/002-python-packages.sh | 40 ++++++++++++ etc/jwstdp-snapshot/tasks/999-clean.sh | 18 ++++++ 24 files changed, 233 insertions(+), 178 deletions(-) delete mode 100644 etc/jwstdp-latest/pip/001-setuptools delete mode 100644 etc/jwstdp-latest/pip/002-testing delete mode 100644 etc/jwstdp-latest/pip/003-prerequisites delete mode 100644 etc/jwstdp-latest/pip/004-pipeline delete mode 100644 etc/jwstdp-latest/pip/005-suppl delete mode 100644 etc/jwstdp-latest/pkgs/000-jwst-depends.sh delete mode 100644 etc/jwstdp-latest/pkgs/001-cfitsio.sh delete mode 100644 etc/jwstdp-latest/pkgs/002-fitsverify.sh delete mode 100755 etc/jwstdp-latest/tasks/001-packages.sh delete mode 100755 etc/jwstdp-latest/tasks/002-python-packages.sh delete mode 100755 etc/jwstdp-latest/tasks/999-clean.sh create mode 100644 etc/jwstdp-snapshot/pip/001-setuptools create mode 100644 etc/jwstdp-snapshot/pip/002-testing create mode 100644 etc/jwstdp-snapshot/pip/003-prerequisites create mode 100644 etc/jwstdp-snapshot/pip/004-pipeline create mode 100644 etc/jwstdp-snapshot/pip/005-suppl create mode 100644 etc/jwstdp-snapshot/pkgs/000-jwst-depends.sh create mode 100644 etc/jwstdp-snapshot/pkgs/001-cfitsio.sh create mode 100644 etc/jwstdp-snapshot/pkgs/002-fitsverify.sh create mode 100755 etc/jwstdp-snapshot/tasks/001-packages.sh create mode 100755 etc/jwstdp-snapshot/tasks/002-python-packages.sh create mode 100755 etc/jwstdp-snapshot/tasks/999-clean.sh diff --git a/Dockerfile b/Dockerfile index b99907b..88bf825 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,6 @@ -ARG PYTHON_VERSION -FROM astroconda/python:${PYTHON_VERSION} +ARG HUB=${HUB:-} +ARG PYTHON_VERSION=${PYTHON_VERSION:-} +FROM "${HUB}/python:${PYTHON_VERSION}" LABEL maintainer="jhunk@stsci.edu" \ vendor="Space Telescope Science Institute" diff --git a/build.sh b/build.sh index b4249bc..cf35562 100755 --- a/build.sh +++ b/build.sh @@ -1,22 +1,76 @@ #!/bin/bash -PROJECT=astroconda/pipeline -PIPELINE="${1}" -if [[ -z ${PIPELINE} ]]; then - echo "No pipeline specified. [e.g. hst-TREE, jwst-TREE, ...]" +set -x +HUB=${3:-astroconda} +PROJECT=${HUB}/pipeline +PROJECT_VERSION="${1}" +PYTHON_VERSION="${2}" +TAGS=() +SUFFIX= +image_tag="${PROJECT_VERSION}" + +if [[ -z ${PROJECT_VERSION} ]]; then + echo "Pipeline version required [e.g. hstdp-2018.3]" exit 1 fi -PYTHON_VERSION="${2}" -if [[ -z ${PIPELINE} ]]; then - echo "Need a fully qualified python version [e.g. 3.7.1]" +if [[ -z ${PYTHON_VERSION} ]]; then + echo "Python version required [e.g. latest, 3.7.2]" exit 1 fi -SUFFIX=${PYTHON_VERSION//\./} -if [[ -z ${SUFFIX} ]]; then - echo "Unable to determine tag suffix from python version." - exit 1 +if [[ ${PYTHON_VERSION} != latest ]]; then + SUFFIX=${PYTHON_VERSION//\./} + if [[ -z ${SUFFIX} ]]; then + echo "Unable to determine tag suffix from python version." + exit 1 + fi + + image_tag="${image_tag}_py${SUFFIX}" fi -TAG="${PROJECT}:${PIPELINE}_py${SUFFIX}" -docker build -t ${TAG} --build-arg PIPELINE=${PIPELINE} --build-arg PYTHON_VERSION=${PYTHON_VERSION} . +case "${HUB}" in + *amazonaws\.com) + if ! type -p aws; then + echo "awscli client not installed" + exit 1 + fi + REGION="$(awk -F'.' '{print $(NF-2)}' <<< ${HUB})" + $(aws ecr get-login --no-include-email --region ${REGION}) + unset REGION + ;; + *) + # Assume default index + docker login + ;; +esac + +TAGS+=( "-t ${PROJECT}:${image_tag}" ) +PIPELINE="${PROJECT_VERSION}" +docker build ${TAGS[@]} \ + --build-arg HUB="${HUB}" \ + --build-arg PYTHON_VERSION="${PYTHON_VERSION}" \ + --build-arg PIPELINE="${PROJECT_VERSION}" \ + . + +rv=$? +if (( rv > 0 )); then + echo "Failed... Image not published" + exit ${rv} +fi + + +max_retry=4 +retry=0 +set +e +while (( retry != max_retry )) +do + echo "Push attempt #$(( retry + 1 ))" + docker push "${PROJECT}:${image_tag}" + rv=$? + if [[ ${rv} == 0 ]]; then + break + fi + (( retry++ )) +done + +exit ${rv} diff --git a/etc/jwstdp-latest/pip/001-setuptools b/etc/jwstdp-latest/pip/001-setuptools deleted file mode 100644 index 648f70c..0000000 --- a/etc/jwstdp-latest/pip/001-setuptools +++ /dev/null @@ -1,2 +0,0 @@ -setuptools>=40.0 -wheel diff --git a/etc/jwstdp-latest/pip/002-testing b/etc/jwstdp-latest/pip/002-testing deleted file mode 100644 index 09a415e..0000000 --- a/etc/jwstdp-latest/pip/002-testing +++ /dev/null @@ -1,6 +0,0 @@ -nose==1.3.7 -pytest>=4.0.0 -pytest-xdist>=1.24.1 -pytest-astropy>=0.5.0 -pytest-virtualenv>=1.3.0 -requests_mock diff --git a/etc/jwstdp-latest/pip/003-prerequisites b/etc/jwstdp-latest/pip/003-prerequisites deleted file mode 100644 index 8ac5fe9..0000000 --- a/etc/jwstdp-latest/pip/003-prerequisites +++ /dev/null @@ -1,4 +0,0 @@ -Cython -numpy>=1.15.2 -relic>=1.1.2 -git+https://github.com/astropy/astropy.git#egg=astropy diff --git a/etc/jwstdp-latest/pip/004-pipeline b/etc/jwstdp-latest/pip/004-pipeline deleted file mode 100644 index 72e47d0..0000000 --- a/etc/jwstdp-latest/pip/004-pipeline +++ /dev/null @@ -1,17 +0,0 @@ -namedlist>=1.7 -parsley>=1.3 -pymssql>=2.1.4 -jplephem>=2.7 -scipy -git+https://github.com/astropy/photutils.git#egg=photutils -git+https://github.com/spacetelescope/asdf.git#egg=asdf -git+https://github.com/spacetelescope/crds.git#egg=crds -git+https://github.com/spacetelescope/drizzle.git#egg=drizzle -git+https://github.com/spacetelescope/gwcs.git#egg=gwcs -git+https://github.com/spacetelescope/stsci.image.git#egg=stsci.image -git+https://github.com/spacetelescope/stsci.stimage.git#egg=stsci.stimage -git+https://github.com/spacetelescope/stsci.imagestats.git#egg=stsci.imagestats -git+https://github.com/spacetelescope/stsci.tools.git#egg=stsci.tools -git+https://github.com/spacetelescope/spherical_geometry.git#egg=spherical_geometry -git+https://github.com/spacetelescope/verhawk.git#egg=verhawk -git+https://github.com/spacetelescope/jwst.git#egg=jwst diff --git a/etc/jwstdp-latest/pip/005-suppl b/etc/jwstdp-latest/pip/005-suppl deleted file mode 100644 index 30317e9..0000000 --- a/etc/jwstdp-latest/pip/005-suppl +++ /dev/null @@ -1 +0,0 @@ -astroquery diff --git a/etc/jwstdp-latest/pkgs/000-jwst-depends.sh b/etc/jwstdp-latest/pkgs/000-jwst-depends.sh deleted file mode 100644 index 88cc61e..0000000 --- a/etc/jwstdp-latest/pkgs/000-jwst-depends.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -deps=( - freetds - unixodbc -) -sudo yum install -y ${deps[@]} diff --git a/etc/jwstdp-latest/pkgs/001-cfitsio.sh b/etc/jwstdp-latest/pkgs/001-cfitsio.sh deleted file mode 100644 index 719b542..0000000 --- a/etc/jwstdp-latest/pkgs/001-cfitsio.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -e - -name=cfitsio -version=3440 -url=https://heasarc.gsfc.nasa.gov/FTP/software/fitsio/c/${name}${version}.tar.gz - -sudo yum install -y libcurl-devel -curl -LO "${url}" -tar xf "$(basename ${url})" - -pushd "${name}" &>/dev/null -./configure --prefix="${PREFIX}" --enable-reentrant -make shared -make install -popd &>/dev/null diff --git a/etc/jwstdp-latest/pkgs/002-fitsverify.sh b/etc/jwstdp-latest/pkgs/002-fitsverify.sh deleted file mode 100644 index 424daf8..0000000 --- a/etc/jwstdp-latest/pkgs/002-fitsverify.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -e - -name=fitsverify -version=4.19 -url=https://heasarc.gsfc.nasa.gov/docs/software/ftools/${name}/${name}-${version}.tar.gz - -curl -LO "${url}" -tar xf "$(basename ${url})" - -pushd "${name}" &>/dev/null -gcc -o ${name} ftverify.c fvrf_data.c fvrf_file.c fvrf_head.c \ - fvrf_key.c fvrf_misc.c ${CFLAGS} -DSTANDALONE \ - ${LDFLAGS} -lcfitsio -lm -lnsl - -install -m 755 -t "${TOOLCHAIN_BIN}" ${name} -popd &>/dev/null diff --git a/etc/jwstdp-latest/tasks/001-packages.sh b/etc/jwstdp-latest/tasks/001-packages.sh deleted file mode 100755 index e64fe06..0000000 --- a/etc/jwstdp-latest/tasks/001-packages.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -set -e -set -x - -sysconfdir="${TOOLCHAIN_BUILD}/etc/${PIPELINE}" -reqdir=${sysconfdir}/pkgs -blddir=builds - - -function pre() -{ - if [[ ! -d ${reqdir} ]]; then - # Nothing there, but maybe that's on purpose. - exit 0 - fi - mkdir -p "${blddir}" - pushd ${blddir} &>/dev/null -} - -function build() -{ - pre - # Iterate over binary package build scripts - for req in ${reqdir}/* - do - chmod +x "${req}" - "${req}" - done - post -} - -function post() -{ - popd &>/dev/null - [[ -d ${blddir} ]] && rm -rf "${blddir}" -} - -build diff --git a/etc/jwstdp-latest/tasks/002-python-packages.sh b/etc/jwstdp-latest/tasks/002-python-packages.sh deleted file mode 100755 index 33714aa..0000000 --- a/etc/jwstdp-latest/tasks/002-python-packages.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -set -e -set -x - -# Uses GLOBAL environment variable: PYTHON_VERSION defined by `docker build` argument -prefix="${TOOLCHAIN}" -sysconfdir="${TOOLCHAIN_BUILD}/etc/${PIPELINE}" -reqdir=${sysconfdir}/pip - -export PATH="${prefix}/bin:${PATH}" -export CFLAGS="-I${prefix}/include" -export LDFLAGS="-L${prefix}/lib -Wl,-rpath=${prefix}/lib" - -function pre() -{ - if [[ ! -d ${reqdir} ]]; then - # Nothing there, but maybe that's on purpose. - exit 0 - fi -} - -function build() -{ - pre - # Iterate over pip requirement files - for req in ${reqdir}/* - do - pip install --upgrade --progress-bar=off -r "${req}" - done - post -} - -function post() -{ - rm -rf ~/.cache/pip - [[ -d src ]] && rm -rf src || true - [[ -f gmon.out ]] && rm -rf gmon.out || true -} - -build diff --git a/etc/jwstdp-latest/tasks/999-clean.sh b/etc/jwstdp-latest/tasks/999-clean.sh deleted file mode 100755 index 7487b9c..0000000 --- a/etc/jwstdp-latest/tasks/999-clean.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -x -if [[ ! -f /.dockerenv ]]; then - echo "This script cannot be executed outside of a docker container." - exit 1 -fi - -rm -rf "${HOME}/.astropy" -rm -rf "${HOME}/.cache" -rm -rf "${HOME}"/* - -sudo yum clean all -sudo rm -rf /tmp/* -sudo rm -rf /var/cache/yum - -for logfile in /var/log/* -do - sudo truncate --size=0 "${logfile}" -done diff --git a/etc/jwstdp-snapshot/pip/001-setuptools b/etc/jwstdp-snapshot/pip/001-setuptools new file mode 100644 index 0000000..648f70c --- /dev/null +++ b/etc/jwstdp-snapshot/pip/001-setuptools @@ -0,0 +1,2 @@ +setuptools>=40.0 +wheel diff --git a/etc/jwstdp-snapshot/pip/002-testing b/etc/jwstdp-snapshot/pip/002-testing new file mode 100644 index 0000000..09a415e --- /dev/null +++ b/etc/jwstdp-snapshot/pip/002-testing @@ -0,0 +1,6 @@ +nose==1.3.7 +pytest>=4.0.0 +pytest-xdist>=1.24.1 +pytest-astropy>=0.5.0 +pytest-virtualenv>=1.3.0 +requests_mock diff --git a/etc/jwstdp-snapshot/pip/003-prerequisites b/etc/jwstdp-snapshot/pip/003-prerequisites new file mode 100644 index 0000000..8ac5fe9 --- /dev/null +++ b/etc/jwstdp-snapshot/pip/003-prerequisites @@ -0,0 +1,4 @@ +Cython +numpy>=1.15.2 +relic>=1.1.2 +git+https://github.com/astropy/astropy.git#egg=astropy diff --git a/etc/jwstdp-snapshot/pip/004-pipeline b/etc/jwstdp-snapshot/pip/004-pipeline new file mode 100644 index 0000000..72e47d0 --- /dev/null +++ b/etc/jwstdp-snapshot/pip/004-pipeline @@ -0,0 +1,17 @@ +namedlist>=1.7 +parsley>=1.3 +pymssql>=2.1.4 +jplephem>=2.7 +scipy +git+https://github.com/astropy/photutils.git#egg=photutils +git+https://github.com/spacetelescope/asdf.git#egg=asdf +git+https://github.com/spacetelescope/crds.git#egg=crds +git+https://github.com/spacetelescope/drizzle.git#egg=drizzle +git+https://github.com/spacetelescope/gwcs.git#egg=gwcs +git+https://github.com/spacetelescope/stsci.image.git#egg=stsci.image +git+https://github.com/spacetelescope/stsci.stimage.git#egg=stsci.stimage +git+https://github.com/spacetelescope/stsci.imagestats.git#egg=stsci.imagestats +git+https://github.com/spacetelescope/stsci.tools.git#egg=stsci.tools +git+https://github.com/spacetelescope/spherical_geometry.git#egg=spherical_geometry +git+https://github.com/spacetelescope/verhawk.git#egg=verhawk +git+https://github.com/spacetelescope/jwst.git#egg=jwst diff --git a/etc/jwstdp-snapshot/pip/005-suppl b/etc/jwstdp-snapshot/pip/005-suppl new file mode 100644 index 0000000..30317e9 --- /dev/null +++ b/etc/jwstdp-snapshot/pip/005-suppl @@ -0,0 +1 @@ +astroquery diff --git a/etc/jwstdp-snapshot/pkgs/000-jwst-depends.sh b/etc/jwstdp-snapshot/pkgs/000-jwst-depends.sh new file mode 100644 index 0000000..88cc61e --- /dev/null +++ b/etc/jwstdp-snapshot/pkgs/000-jwst-depends.sh @@ -0,0 +1,6 @@ +#!/bin/bash +deps=( + freetds + unixodbc +) +sudo yum install -y ${deps[@]} diff --git a/etc/jwstdp-snapshot/pkgs/001-cfitsio.sh b/etc/jwstdp-snapshot/pkgs/001-cfitsio.sh new file mode 100644 index 0000000..719b542 --- /dev/null +++ b/etc/jwstdp-snapshot/pkgs/001-cfitsio.sh @@ -0,0 +1,15 @@ +#!/bin/bash -e + +name=cfitsio +version=3440 +url=https://heasarc.gsfc.nasa.gov/FTP/software/fitsio/c/${name}${version}.tar.gz + +sudo yum install -y libcurl-devel +curl -LO "${url}" +tar xf "$(basename ${url})" + +pushd "${name}" &>/dev/null +./configure --prefix="${PREFIX}" --enable-reentrant +make shared +make install +popd &>/dev/null diff --git a/etc/jwstdp-snapshot/pkgs/002-fitsverify.sh b/etc/jwstdp-snapshot/pkgs/002-fitsverify.sh new file mode 100644 index 0000000..424daf8 --- /dev/null +++ b/etc/jwstdp-snapshot/pkgs/002-fitsverify.sh @@ -0,0 +1,16 @@ +#!/bin/bash -e + +name=fitsverify +version=4.19 +url=https://heasarc.gsfc.nasa.gov/docs/software/ftools/${name}/${name}-${version}.tar.gz + +curl -LO "${url}" +tar xf "$(basename ${url})" + +pushd "${name}" &>/dev/null +gcc -o ${name} ftverify.c fvrf_data.c fvrf_file.c fvrf_head.c \ + fvrf_key.c fvrf_misc.c ${CFLAGS} -DSTANDALONE \ + ${LDFLAGS} -lcfitsio -lm -lnsl + +install -m 755 -t "${TOOLCHAIN_BIN}" ${name} +popd &>/dev/null diff --git a/etc/jwstdp-snapshot/tasks/001-packages.sh b/etc/jwstdp-snapshot/tasks/001-packages.sh new file mode 100755 index 0000000..e64fe06 --- /dev/null +++ b/etc/jwstdp-snapshot/tasks/001-packages.sh @@ -0,0 +1,38 @@ +#!/bin/bash +set -e +set -x + +sysconfdir="${TOOLCHAIN_BUILD}/etc/${PIPELINE}" +reqdir=${sysconfdir}/pkgs +blddir=builds + + +function pre() +{ + if [[ ! -d ${reqdir} ]]; then + # Nothing there, but maybe that's on purpose. + exit 0 + fi + mkdir -p "${blddir}" + pushd ${blddir} &>/dev/null +} + +function build() +{ + pre + # Iterate over binary package build scripts + for req in ${reqdir}/* + do + chmod +x "${req}" + "${req}" + done + post +} + +function post() +{ + popd &>/dev/null + [[ -d ${blddir} ]] && rm -rf "${blddir}" +} + +build diff --git a/etc/jwstdp-snapshot/tasks/002-python-packages.sh b/etc/jwstdp-snapshot/tasks/002-python-packages.sh new file mode 100755 index 0000000..33714aa --- /dev/null +++ b/etc/jwstdp-snapshot/tasks/002-python-packages.sh @@ -0,0 +1,40 @@ +#!/bin/bash +set -e +set -x + +# Uses GLOBAL environment variable: PYTHON_VERSION defined by `docker build` argument +prefix="${TOOLCHAIN}" +sysconfdir="${TOOLCHAIN_BUILD}/etc/${PIPELINE}" +reqdir=${sysconfdir}/pip + +export PATH="${prefix}/bin:${PATH}" +export CFLAGS="-I${prefix}/include" +export LDFLAGS="-L${prefix}/lib -Wl,-rpath=${prefix}/lib" + +function pre() +{ + if [[ ! -d ${reqdir} ]]; then + # Nothing there, but maybe that's on purpose. + exit 0 + fi +} + +function build() +{ + pre + # Iterate over pip requirement files + for req in ${reqdir}/* + do + pip install --upgrade --progress-bar=off -r "${req}" + done + post +} + +function post() +{ + rm -rf ~/.cache/pip + [[ -d src ]] && rm -rf src || true + [[ -f gmon.out ]] && rm -rf gmon.out || true +} + +build diff --git a/etc/jwstdp-snapshot/tasks/999-clean.sh b/etc/jwstdp-snapshot/tasks/999-clean.sh new file mode 100755 index 0000000..7487b9c --- /dev/null +++ b/etc/jwstdp-snapshot/tasks/999-clean.sh @@ -0,0 +1,18 @@ +#!/bin/bash -x +if [[ ! -f /.dockerenv ]]; then + echo "This script cannot be executed outside of a docker container." + exit 1 +fi + +rm -rf "${HOME}/.astropy" +rm -rf "${HOME}/.cache" +rm -rf "${HOME}"/* + +sudo yum clean all +sudo rm -rf /tmp/* +sudo rm -rf /var/cache/yum + +for logfile in /var/log/* +do + sudo truncate --size=0 "${logfile}" +done -- cgit