From 92f2ea6ed46ffae5302a47452d816b5dc5784eb7 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Mon, 28 Jan 2019 12:57:22 -0500 Subject: Reorder tools --- etc/tasks/001-packages.sh | 43 ++++++++++++++++++++++++++++++++++++++++ etc/tasks/002-python-packages.sh | 39 ++++++++++++++++++++++++++++++++++++ etc/tasks/998-debase.sh | 7 +++++++ 3 files changed, 89 insertions(+) create mode 100755 etc/tasks/001-packages.sh create mode 100755 etc/tasks/002-python-packages.sh create mode 100755 etc/tasks/998-debase.sh (limited to 'etc/tasks') diff --git a/etc/tasks/001-packages.sh b/etc/tasks/001-packages.sh new file mode 100755 index 0000000..f7d0cad --- /dev/null +++ b/etc/tasks/001-packages.sh @@ -0,0 +1,43 @@ +#!/bin/bash +set -e +set -x + +prefix="${TOOLCHAIN}" +sysconfdir="${TOOLCHAIN_BUILD}/etc" +reqdir=${sysconfdir}/pkgs +blddir=builds + +export PATH="${prefix}/bin:${PATH}" +export CFLAGS="-I${prefix}/include" +export LDFLAGS="-L${prefix}/lib -Wl,-rpath=${prefix}/lib" +export PREFIX="${prefix}" + +function pre() +{ + if [[ ! -d ${reqdir} ]]; then + # Nothing there, but maybe that's on purpose. + exit 0 + fi + mkdir -p "${blddir}" + pushd ${blddir} &>/dev/null +} + +function build() +{ + pre + # Iterate over binary package build scripts + for req in ${reqdir}/* + do + chmod +x "${req}" + "${req}" + done + post +} + +function post() +{ + popd &>/dev/null + [[ -d ${blddir} ]] && rm -rf "${blddir}" +} + +build diff --git a/etc/tasks/002-python-packages.sh b/etc/tasks/002-python-packages.sh new file mode 100755 index 0000000..cb248ff --- /dev/null +++ b/etc/tasks/002-python-packages.sh @@ -0,0 +1,39 @@ +#!/bin/bash +set -x + +# Uses GLOBAL environment variable: PYTHON_VERSION defined by `docker build` argument +prefix="${TOOLCHAIN}" +sysconfdir="${TOOLCHAIN_BUILD}/etc" +reqdir=${sysconfdir}/pip + +export PATH="${prefix}/bin:${PATH}" +export CFLAGS="-I${prefix}/include" +export LDFLAGS="-L${prefix}/lib -Wl,-rpath=${prefix}/lib" + +function pre() +{ + if [[ ! -d ${reqdir} ]]; then + # Nothing there, but maybe that's on purpose. + exit 0 + fi +} + +function build() +{ + pre + # Iterate over pip requirement files + for req in ${reqdir}/* + do + pip install --upgrade --progress-bar=off -v -r "${req}" + done + post +} + +function post() +{ + rm -rf ~/.cache/pip + [[ -d src ]] && rm -rf src + [[ -f gmon.out ]] && rm -rf gmon.out +} + +build diff --git a/etc/tasks/998-debase.sh b/etc/tasks/998-debase.sh new file mode 100755 index 0000000..8b56cd4 --- /dev/null +++ b/etc/tasks/998-debase.sh @@ -0,0 +1,7 @@ +#!/bin/bash +packages=( + gcc + gcc-c++ + gcc-gfortran +) +yum remove -y "${packages[@]}" -- cgit From 73a53c48901e120a06d221c8cb4f02076e3b9888 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Tue, 29 Jan 2019 14:38:49 -0500 Subject: Introduce real toolchain --- etc/tasks/001-openssl.sh | 73 --------------------- etc/tasks/001-packages.sh | 6 ++ etc/tasks/002-python-packages.sh | 12 +++- etc/tasks/002-python.sh | 133 --------------------------------------- etc/tasks/003-python-packages.sh | 39 ------------ etc/tasks/004-packages.sh | 43 ------------- etc/tasks/998-debase.sh | 7 --- etc/tasks/999-clean.sh | 9 +++ 8 files changed, 24 insertions(+), 298 deletions(-) delete mode 100755 etc/tasks/001-openssl.sh delete mode 100755 etc/tasks/002-python.sh delete mode 100755 etc/tasks/003-python-packages.sh delete mode 100755 etc/tasks/004-packages.sh delete mode 100755 etc/tasks/998-debase.sh (limited to 'etc/tasks') diff --git a/etc/tasks/001-openssl.sh b/etc/tasks/001-openssl.sh deleted file mode 100755 index 1f99fe8..0000000 --- a/etc/tasks/001-openssl.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash -set -e -set -x - -name="openssl" -version="1.1.0j" - -tarball="${name}-${version}.tar.gz" -dest="${tarball%%.tar.gz}" -url="https://www.openssl.org/source/${tarball}" -prefix="${TOOLCHAIN}" - - -function pre() -{ - curl -LO "${url}" - tar xf "${tarball}" -} - - -function get_system_cacert() { - local paths=( - /etc/ssl/cert.pem - /etc/ssl/cacert.pem - /etc/ssl/certs/cacert.pem - /etc/ssl/certs/ca-bundle.crt - ) - for bundle in "${paths[@]}" - do - if [[ -f ${bundle} ]]; then - echo "${bundle}" - break - fi - done -} - - -function build() -{ - pre - pushd "${dest}" - export PATH="${prefix}/bin:${PATH}" - export LDFLAGS="-Wl,-rpath=${prefix}/lib" - export KERNEL_BITS=64 - target="linux-x86_64" - - ./Configure \ - --prefix="${prefix}" \ - --openssldir="ssl" \ - --libdir="lib" \ - ${LDFLAGS} \ - ${target} \ - enable-ec_nistp_64_gcc_128 \ - zlib-dynamic \ - shared \ - no-ssl3-method - make - make install MANDIR="${prefix}/share/man" MANSUFFIX=ssl - popd - post -} - -function post() -{ - bundle=$(get_system_cacert) - install -D -m644 "${bundle}" "${prefix}/ssl/cert.pem" - rm -rf "${dest}" - rm -rf "${tarball}" - echo "All done." -} - -# Main -build diff --git a/etc/tasks/001-packages.sh b/etc/tasks/001-packages.sh index f7d0cad..fabad6e 100755 --- a/etc/tasks/001-packages.sh +++ b/etc/tasks/001-packages.sh @@ -10,6 +10,7 @@ blddir=builds export PATH="${prefix}/bin:${PATH}" export CFLAGS="-I${prefix}/include" export LDFLAGS="-L${prefix}/lib -Wl,-rpath=${prefix}/lib" +export PKG_CONFIG_PATH="${prefix}/lib/pkgconfig" export PREFIX="${prefix}" function pre() @@ -30,6 +31,11 @@ function build() do chmod +x "${req}" "${req}" + retval=$? + if [[ ${retval} != 0 ]]; then + echo "BUILD FAILED: ${req}" + exit ${retval} + fi done post } diff --git a/etc/tasks/002-python-packages.sh b/etc/tasks/002-python-packages.sh index cb248ff..9218a8e 100755 --- a/etc/tasks/002-python-packages.sh +++ b/etc/tasks/002-python-packages.sh @@ -7,6 +7,7 @@ sysconfdir="${TOOLCHAIN_BUILD}/etc" reqdir=${sysconfdir}/pip export PATH="${prefix}/bin:${PATH}" +export PKG_CONFIG_PATH="${prefix}/lib/pkgconfig" export CFLAGS="-I${prefix}/include" export LDFLAGS="-L${prefix}/lib -Wl,-rpath=${prefix}/lib" @@ -24,7 +25,12 @@ function build() # Iterate over pip requirement files for req in ${reqdir}/* do - pip install --upgrade --progress-bar=off -v -r "${req}" + pip install --upgrade --progress-bar=off -r "${req}" + retval=$? + if [[ ${retval} != 0 ]]; then + echo "BUILD FAILED: ${req}" + exit ${retval} + fi done post } @@ -32,8 +38,8 @@ function build() function post() { rm -rf ~/.cache/pip - [[ -d src ]] && rm -rf src - [[ -f gmon.out ]] && rm -rf gmon.out + [[ -d src ]] && rm -rf src || true + [[ -f gmon.out ]] && rm -rf gmon.out || true } build diff --git a/etc/tasks/002-python.sh b/etc/tasks/002-python.sh deleted file mode 100755 index d7db12f..0000000 --- a/etc/tasks/002-python.sh +++ /dev/null @@ -1,133 +0,0 @@ -#!/bin/bash -set -e -set -x - -python_version="${PYTHON_VERSION}" -python_basever="${python_version%.*}" - -if [[ ! ${python_version} || ! ${python_basever} ]]; then - echo "Need a python version..." - exit 1 -fi - -python_base_url="https://www.python.org/ftp/python" -python_tarball="Python-${python_version}.tgz" -python_source="${python_tarball%%.tgz}" -python_url="${python_base_url}/${python_version}/${python_tarball}" -prefix="${TOOLCHAIN}" - -dep_table=( - "bzlib.h libbz2.so" - "expat.h libexpat.so" - "ffi.h libffi.so" - "gdbm.h libgdbm.so" - "lzma.h liblzma.so" - "ncurses.h libncurses.so" - "nislib.h libnsl.so" - "readline.h libreadline.so" - "ssl.h libssl.so" - "sqlite3.h libsqlite3.so" - "tcl.h libtcl.so" - "tk.h libtk.so" - "zlib.h libz.so" -) - - -function depcheck() -{ - dep_count=0 - dep_total="${#dep_table[@]}" - - set +x - for _record in "${dep_table[@]}" - do - unset record - read -ra record <<< $_record - - header=$(find /usr/include /usr/lib{,64} -regex ".*\/${record[0]}" 2>/dev/null | head -n 1 || true) - if [[ -n $header ]]; then - dep_count=$((dep_count+1)) - else - echo "Missing header: ${record[0]}" - fi - lib=$(find /usr/lib{,64} -regex ".*\/${record[1]}" 2>/dev/null | head -n 1 || true) - if [[ -n "$lib" ]]; then - dep_count=$((dep_count+1)) - else - echo "Missing library: ${record[1]}" - fi - done - set -x - - if [[ ${dep_count} != $(( (dep_total * 2) )) ]]; then - echo 'Missing dependencies...' - exit 1 - fi -} - - -function pre() -{ - depcheck - - if [[ ! -f ${python_tarball} ]]; then - curl -LO "${python_url}" - fi - - if [[ -d ${python_source} ]]; then - rm -rf "${python_source}" - fi - - tar xf "${python_tarball}" -} - - -function build() -{ - pre - export CFLAGS="-I${prefix}/include" - export LDFLAGS="-L${prefix}/lib -Wl,-rpath=${prefix}/lib" - pushd "${python_source}" - #--enable-optimizations \ - ./configure \ - --prefix="${prefix}" \ - --enable-ipv6 \ - --enable-loadable-sqlite-extensions \ - --enable-profiling \ - --enable-shared \ - --with-dbmliborder=gdbm:ndbm \ - --with-pymalloc \ - --with-system-expat - make -j4 - make install - popd - post -} - - -function post() -{ - export PATH=$prefix/bin:$PATH - ln -sf python3 "${prefix}"/bin/python - ln -sf python3-config "${prefix}"/bin/python-config - ln -sf idle3 "${prefix}"/bin/idle - ln -sf pydoc3 "${prefix}"/bin/pydoc - ln -sf pip3 "${prefix}"/bin/pip - ln -sf python${python_basever}.1 "${prefix}"/share/man/man1/python.1 - - echo '---' - python --version - python -c "import sys; from pprint import pprint; pprint(sys.path)" - echo '---' - ldd $(which python) - echo '---' - - rm -rf $HOME/.config/pip - rm -rf "${python_tarball}" - rm -rf "${python_source}" - echo "All done." -} - - -# Main -build diff --git a/etc/tasks/003-python-packages.sh b/etc/tasks/003-python-packages.sh deleted file mode 100755 index cb248ff..0000000 --- a/etc/tasks/003-python-packages.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -set -x - -# Uses GLOBAL environment variable: PYTHON_VERSION defined by `docker build` argument -prefix="${TOOLCHAIN}" -sysconfdir="${TOOLCHAIN_BUILD}/etc" -reqdir=${sysconfdir}/pip - -export PATH="${prefix}/bin:${PATH}" -export CFLAGS="-I${prefix}/include" -export LDFLAGS="-L${prefix}/lib -Wl,-rpath=${prefix}/lib" - -function pre() -{ - if [[ ! -d ${reqdir} ]]; then - # Nothing there, but maybe that's on purpose. - exit 0 - fi -} - -function build() -{ - pre - # Iterate over pip requirement files - for req in ${reqdir}/* - do - pip install --upgrade --progress-bar=off -v -r "${req}" - done - post -} - -function post() -{ - rm -rf ~/.cache/pip - [[ -d src ]] && rm -rf src - [[ -f gmon.out ]] && rm -rf gmon.out -} - -build diff --git a/etc/tasks/004-packages.sh b/etc/tasks/004-packages.sh deleted file mode 100755 index f7d0cad..0000000 --- a/etc/tasks/004-packages.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -set -e -set -x - -prefix="${TOOLCHAIN}" -sysconfdir="${TOOLCHAIN_BUILD}/etc" -reqdir=${sysconfdir}/pkgs -blddir=builds - -export PATH="${prefix}/bin:${PATH}" -export CFLAGS="-I${prefix}/include" -export LDFLAGS="-L${prefix}/lib -Wl,-rpath=${prefix}/lib" -export PREFIX="${prefix}" - -function pre() -{ - if [[ ! -d ${reqdir} ]]; then - # Nothing there, but maybe that's on purpose. - exit 0 - fi - mkdir -p "${blddir}" - pushd ${blddir} &>/dev/null -} - -function build() -{ - pre - # Iterate over binary package build scripts - for req in ${reqdir}/* - do - chmod +x "${req}" - "${req}" - done - post -} - -function post() -{ - popd &>/dev/null - [[ -d ${blddir} ]] && rm -rf "${blddir}" -} - -build diff --git a/etc/tasks/998-debase.sh b/etc/tasks/998-debase.sh deleted file mode 100755 index 8b56cd4..0000000 --- a/etc/tasks/998-debase.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -packages=( - gcc - gcc-c++ - gcc-gfortran -) -yum remove -y "${packages[@]}" diff --git a/etc/tasks/999-clean.sh b/etc/tasks/999-clean.sh index 44f5d16..c791ec5 100755 --- a/etc/tasks/999-clean.sh +++ b/etc/tasks/999-clean.sh @@ -4,8 +4,17 @@ if [[ ! -f /.dockerenv ]]; then exit 1 fi +packages=( + gcc + gcc-c++ + gcc-gfortran +) +sudo yum remove -y "${packages[@]}" sudo yum clean all +# Remove all static libraries +sudo find "${TOOLCHAIN_LIB}" -name '*.a' -delete + sudo rm -rf "${HOME}/.astropy" sudo rm -rf "${HOME}"/* sudo rm -rf /tmp/* -- cgit From 11f358cafab2da28692fdaa837dfbcc3cc163736 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Thu, 31 Jan 2019 14:12:13 -0500 Subject: Refactor all --- etc/tasks/001-packages.sh | 8 -------- etc/tasks/002-python-packages.sh | 7 ------- etc/tasks/999-clean.sh | 12 +++--------- 3 files changed, 3 insertions(+), 24 deletions(-) (limited to 'etc/tasks') diff --git a/etc/tasks/001-packages.sh b/etc/tasks/001-packages.sh index fabad6e..5d20ee7 100755 --- a/etc/tasks/001-packages.sh +++ b/etc/tasks/001-packages.sh @@ -1,18 +1,10 @@ #!/bin/bash -set -e set -x -prefix="${TOOLCHAIN}" sysconfdir="${TOOLCHAIN_BUILD}/etc" reqdir=${sysconfdir}/pkgs blddir=builds -export PATH="${prefix}/bin:${PATH}" -export CFLAGS="-I${prefix}/include" -export LDFLAGS="-L${prefix}/lib -Wl,-rpath=${prefix}/lib" -export PKG_CONFIG_PATH="${prefix}/lib/pkgconfig" -export PREFIX="${prefix}" - function pre() { if [[ ! -d ${reqdir} ]]; then diff --git a/etc/tasks/002-python-packages.sh b/etc/tasks/002-python-packages.sh index 9218a8e..bcd0b6f 100755 --- a/etc/tasks/002-python-packages.sh +++ b/etc/tasks/002-python-packages.sh @@ -1,16 +1,9 @@ #!/bin/bash set -x -# Uses GLOBAL environment variable: PYTHON_VERSION defined by `docker build` argument -prefix="${TOOLCHAIN}" sysconfdir="${TOOLCHAIN_BUILD}/etc" reqdir=${sysconfdir}/pip -export PATH="${prefix}/bin:${PATH}" -export PKG_CONFIG_PATH="${prefix}/lib/pkgconfig" -export CFLAGS="-I${prefix}/include" -export LDFLAGS="-L${prefix}/lib -Wl,-rpath=${prefix}/lib" - function pre() { if [[ ! -d ${reqdir} ]]; then diff --git a/etc/tasks/999-clean.sh b/etc/tasks/999-clean.sh index c791ec5..e4e8d13 100755 --- a/etc/tasks/999-clean.sh +++ b/etc/tasks/999-clean.sh @@ -4,22 +4,16 @@ if [[ ! -f /.dockerenv ]]; then exit 1 fi -packages=( - gcc - gcc-c++ - gcc-gfortran -) -sudo yum remove -y "${packages[@]}" sudo yum clean all -# Remove all static libraries -sudo find "${TOOLCHAIN_LIB}" -name '*.a' -delete - sudo rm -rf "${HOME}/.astropy" sudo rm -rf "${HOME}"/* sudo rm -rf /tmp/* sudo rm -rf /var/cache/yum +# Ensure new shared libraries ingested +sudo ldconfig + for logfile in /var/log/* do [[ -f ${logfile} ]] && sudo truncate --size=0 "${logfile}" -- cgit