From 73a53c48901e120a06d221c8cb4f02076e3b9888 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Tue, 29 Jan 2019 14:38:49 -0500 Subject: Introduce real toolchain --- Dockerfile | 63 +++++++++---------- etc/pkgs/000-base.sh | 27 -------- etc/pkgs/000-dummy.sh | 0 etc/pkgs/001-gcc.sh | 9 ++- etc/pkgs/002-binutils.sh | 18 ++++-- etc/pkgs/003-openssh.sh | 73 --------------------- etc/pkgs/003-openssl.sh | 73 +++++++++++++++++++++ etc/tasks/001-openssl.sh | 73 --------------------- etc/tasks/001-packages.sh | 6 ++ etc/tasks/002-python-packages.sh | 12 +++- etc/tasks/002-python.sh | 133 --------------------------------------- etc/tasks/003-python-packages.sh | 39 ------------ etc/tasks/004-packages.sh | 43 ------------- etc/tasks/998-debase.sh | 7 --- etc/tasks/999-clean.sh | 9 +++ scripts/build.sh | 6 ++ 16 files changed, 154 insertions(+), 437 deletions(-) delete mode 100755 etc/pkgs/000-base.sh mode change 100644 => 100755 etc/pkgs/000-dummy.sh delete mode 100755 etc/pkgs/003-openssh.sh create mode 100755 etc/pkgs/003-openssl.sh delete mode 100755 etc/tasks/001-openssl.sh delete mode 100755 etc/tasks/002-python.sh delete mode 100755 etc/tasks/003-python-packages.sh delete mode 100755 etc/tasks/004-packages.sh delete mode 100755 etc/tasks/998-debase.sh diff --git a/Dockerfile b/Dockerfile index 04e29c3..82214fd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,35 +2,6 @@ FROM centos:6.9 LABEL maintainer="jhunk@stsci.edu" \ vendor="Space Telescope Science Institute" -RUN yum install -y epel-release \ - && yum clean all - -RUN yum install -y \ - gcc \ - gcc-c++ \ - gcc-gfortran \ - git \ - glibc \ - libuuid-devel \ - make \ - perl \ - pkgconfig \ - expat-devel \ - bzip2-devel \ - gdbm-devel \ - libffi-devel \ - ncurses-devel \ - openssl-devel \ - readline-devel \ - sqlite-devel \ - sudo \ - tcl-devel \ - tk-devel \ - which \ - xz-devel \ - zlib-devel \ - && yum clean all - ENV TOOLCHAIN="/opt/toolchain" ENV TOOLCHAIN_BIN="${TOOLCHAIN}/bin" ENV TOOLCHAIN_LIB="${TOOLCHAIN}/lib" @@ -48,13 +19,39 @@ ARG USER_HOME=/home/${USER_ACCT} ARG USER_UID=${USER_UID:-1000} ARG USER_GID=${USER_GID:-1000} -RUN groupadd -g ${USER_GID} ${USER_ACCT} \ +RUN yum install -y epel-release \ + && yum install -y \ + gcc \ + gcc-c++ \ + gcc-gfortran \ + git \ + glibc \ + libuuid-devel \ + make \ + perl \ + pkgconfig \ + expat-devel \ + bzip2-devel \ + gdbm-devel \ + libffi-devel \ + ncurses-devel \ + openssl-devel \ + readline-devel \ + sqlite-devel \ + sudo \ + tcl-devel \ + tk-devel \ + wget \ + which \ + xz-devel \ + zlib-devel \ + && yum clean all \ + && groupadd -g ${USER_GID} ${USER_ACCT} \ && useradd -u ${USER_UID} -g ${USER_ACCT} \ -m -d ${USER_HOME} -s /bin/bash ${USER_ACCT} \ && echo "${USER_ACCT}:${USER_ACCT}" | chpasswd \ - && echo "${USER_ACCT} ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers - -RUN echo export PATH="${TOOLCHAIN_BIN}:\${PATH}" > /etc/profile.d/toolchain.sh \ + && echo "${USER_ACCT} ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers \ + && echo export PATH="${TOOLCHAIN_BIN}:\${PATH}" > /etc/profile.d/toolchain.sh \ && echo export MANPATH="${TOOLCHAIN_MAN}:\${MANPATH}" >> /etc/profile.d/toolchain.sh \ && echo export PKG_CONFIG_PATH="${TOOLCHAIN_PKGCONFIG}:\${PKG_CONFIG_PATH}" >> /etc/profile.d/toolchain.sh diff --git a/etc/pkgs/000-base.sh b/etc/pkgs/000-base.sh deleted file mode 100755 index e32bbae..0000000 --- a/etc/pkgs/000-base.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -packages=( - gcc - gcc-c++ - gcc-gfortran - git - glibc - libuuid-devel - make - perl - pkgconfig - expat-devel - bzip2-devel - gdbm-devel - libffi-devel - ncurses-devel - openssl-devel - readline-devel - sqlite-devel - sudo - tcl-devel - tk-devel - which - xz-devel - zlib-devel -) -yum install -y "${packages[@]}" diff --git a/etc/pkgs/000-dummy.sh b/etc/pkgs/000-dummy.sh old mode 100644 new mode 100755 diff --git a/etc/pkgs/001-gcc.sh b/etc/pkgs/001-gcc.sh index 52c01da..21065cd 100755 --- a/etc/pkgs/001-gcc.sh +++ b/etc/pkgs/001-gcc.sh @@ -12,7 +12,7 @@ url_isl=http://isl.gforge.inria.fr/isl-${version_isl}.tar.bz2 version_cloog=0.18.4 url_cloog="http://www.bastoul.net/cloog/pages/download/count.php3?url=./cloog-${version_cloog}.tar.gz" -sudo yum install -y wget +# Ensure 64-bit libraries go into /lib sudo ln -sf ${TOOLCHAIN_LIB} ${TOOLCHAIN}/lib64 curl -LO ${url} @@ -49,7 +49,7 @@ pushd ${bld} --prefix=${TOOLCHAIN} \ --libdir=${TOOLCHAIN_LIB} \ --libexecdir=${TOOLCHAIN_LIB} \ - --disable-bootstrap \ + --disable-static \ --disable-multilib \ --disable-werror \ --disable-libunwind-exceptions \ @@ -58,6 +58,8 @@ pushd ${bld} --with-system-zlib \ --with-isl \ --with-linker-hash-style=gnu \ + --with-tune=generic \ + --enable-bootstrap \ --enable-languages=c,c++,fortran,lto,go \ --enable-shared \ --enable-threads=posix \ @@ -78,6 +80,9 @@ pushd ${bld} make -j${_maxjobs} make install-strip + # Binutils build cannot use this static archive + rm -f "${TOOLCHAIN_LIB}/libiberty.a" + # Prevent ldconfig from picking up gdb python scripts autoload="${TOOLCHAIN_DATA}/gdb/auto-load${TOOLCHAIN_LIB}" mkdir -p "${autoload}" diff --git a/etc/pkgs/002-binutils.sh b/etc/pkgs/002-binutils.sh index bd119fe..83423be 100755 --- a/etc/pkgs/002-binutils.sh +++ b/etc/pkgs/002-binutils.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -x name=binutils version=2.31.1 url=https://ftp.gnu.org/gnu/binutils/${name}-${version}.tar.gz @@ -10,7 +10,17 @@ mkdir -p binutils pushd binutils ../${name}-${version}/configure \ --prefix=${TOOLCHAIN} \ - --with-sysroot=${TOOLCHAIN} - make -j${_maxjobs} - make install + --target=x86_64-pc-linux-gnu \ + --enable-shared \ + --enable-lto \ + --enable-gold \ + --enable-ld=default \ + --enable-plugins \ + --enable-threads \ + --disable-static \ + --disable-multilib \ + --with-sysroot=/ \ + --with-tune=generic + make -j4 + make install-strip popd diff --git a/etc/pkgs/003-openssh.sh b/etc/pkgs/003-openssh.sh deleted file mode 100755 index 1f99fe8..0000000 --- a/etc/pkgs/003-openssh.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash -set -e -set -x - -name="openssl" -version="1.1.0j" - -tarball="${name}-${version}.tar.gz" -dest="${tarball%%.tar.gz}" -url="https://www.openssl.org/source/${tarball}" -prefix="${TOOLCHAIN}" - - -function pre() -{ - curl -LO "${url}" - tar xf "${tarball}" -} - - -function get_system_cacert() { - local paths=( - /etc/ssl/cert.pem - /etc/ssl/cacert.pem - /etc/ssl/certs/cacert.pem - /etc/ssl/certs/ca-bundle.crt - ) - for bundle in "${paths[@]}" - do - if [[ -f ${bundle} ]]; then - echo "${bundle}" - break - fi - done -} - - -function build() -{ - pre - pushd "${dest}" - export PATH="${prefix}/bin:${PATH}" - export LDFLAGS="-Wl,-rpath=${prefix}/lib" - export KERNEL_BITS=64 - target="linux-x86_64" - - ./Configure \ - --prefix="${prefix}" \ - --openssldir="ssl" \ - --libdir="lib" \ - ${LDFLAGS} \ - ${target} \ - enable-ec_nistp_64_gcc_128 \ - zlib-dynamic \ - shared \ - no-ssl3-method - make - make install MANDIR="${prefix}/share/man" MANSUFFIX=ssl - popd - post -} - -function post() -{ - bundle=$(get_system_cacert) - install -D -m644 "${bundle}" "${prefix}/ssl/cert.pem" - rm -rf "${dest}" - rm -rf "${tarball}" - echo "All done." -} - -# Main -build diff --git a/etc/pkgs/003-openssl.sh b/etc/pkgs/003-openssl.sh new file mode 100755 index 0000000..c5aaabe --- /dev/null +++ b/etc/pkgs/003-openssl.sh @@ -0,0 +1,73 @@ +#!/bin/bash +set -e +set -x + +name="openssl" +version="1.1.0j" + +tarball="${name}-${version}.tar.gz" +dest="${tarball%%.tar.gz}" +url="https://www.openssl.org/source/${tarball}" +prefix="${TOOLCHAIN}" + + +function pre() +{ + curl -LO "${url}" + tar xf "${tarball}" +} + + +function get_system_cacert() { + local paths=( + /etc/ssl/cert.pem + /etc/ssl/cacert.pem + /etc/ssl/certs/cacert.pem + /etc/ssl/certs/ca-bundle.crt + ) + for bundle in "${paths[@]}" + do + if [[ -f ${bundle} ]]; then + echo "${bundle}" + break + fi + done +} + + +function build() +{ + pre + pushd "${dest}" + export PATH="${prefix}/bin:${PATH}" + export LDFLAGS="-Wl,-rpath=${prefix}/lib" + export KERNEL_BITS=64 + target="linux-x86_64" + + ./Configure \ + --prefix="${prefix}" \ + --openssldir="ssl" \ + --libdir="lib" \ + ${LDFLAGS} \ + ${target} \ + enable-ec_nistp_64_gcc_128 \ + zlib-dynamic \ + shared \ + no-ssl3-method + make -j${_maxjobs} + make install MANDIR="${prefix}/share/man" MANSUFFIX=ssl + popd + post +} + +function post() +{ + bundle=$(get_system_cacert) + install -D -m644 "${bundle}" "${prefix}/ssl/cert.pem" + rm -rf "${dest}" + rm -rf "${tarball}" + echo "All done." +} + +# Main +build diff --git a/etc/tasks/001-openssl.sh b/etc/tasks/001-openssl.sh deleted file mode 100755 index 1f99fe8..0000000 --- a/etc/tasks/001-openssl.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash -set -e -set -x - -name="openssl" -version="1.1.0j" - -tarball="${name}-${version}.tar.gz" -dest="${tarball%%.tar.gz}" -url="https://www.openssl.org/source/${tarball}" -prefix="${TOOLCHAIN}" - - -function pre() -{ - curl -LO "${url}" - tar xf "${tarball}" -} - - -function get_system_cacert() { - local paths=( - /etc/ssl/cert.pem - /etc/ssl/cacert.pem - /etc/ssl/certs/cacert.pem - /etc/ssl/certs/ca-bundle.crt - ) - for bundle in "${paths[@]}" - do - if [[ -f ${bundle} ]]; then - echo "${bundle}" - break - fi - done -} - - -function build() -{ - pre - pushd "${dest}" - export PATH="${prefix}/bin:${PATH}" - export LDFLAGS="-Wl,-rpath=${prefix}/lib" - export KERNEL_BITS=64 - target="linux-x86_64" - - ./Configure \ - --prefix="${prefix}" \ - --openssldir="ssl" \ - --libdir="lib" \ - ${LDFLAGS} \ - ${target} \ - enable-ec_nistp_64_gcc_128 \ - zlib-dynamic \ - shared \ - no-ssl3-method - make - make install MANDIR="${prefix}/share/man" MANSUFFIX=ssl - popd - post -} - -function post() -{ - bundle=$(get_system_cacert) - install -D -m644 "${bundle}" "${prefix}/ssl/cert.pem" - rm -rf "${dest}" - rm -rf "${tarball}" - echo "All done." -} - -# Main -build diff --git a/etc/tasks/001-packages.sh b/etc/tasks/001-packages.sh index f7d0cad..fabad6e 100755 --- a/etc/tasks/001-packages.sh +++ b/etc/tasks/001-packages.sh @@ -10,6 +10,7 @@ blddir=builds export PATH="${prefix}/bin:${PATH}" export CFLAGS="-I${prefix}/include" export LDFLAGS="-L${prefix}/lib -Wl,-rpath=${prefix}/lib" +export PKG_CONFIG_PATH="${prefix}/lib/pkgconfig" export PREFIX="${prefix}" function pre() @@ -30,6 +31,11 @@ function build() do chmod +x "${req}" "${req}" + retval=$? + if [[ ${retval} != 0 ]]; then + echo "BUILD FAILED: ${req}" + exit ${retval} + fi done post } diff --git a/etc/tasks/002-python-packages.sh b/etc/tasks/002-python-packages.sh index cb248ff..9218a8e 100755 --- a/etc/tasks/002-python-packages.sh +++ b/etc/tasks/002-python-packages.sh @@ -7,6 +7,7 @@ sysconfdir="${TOOLCHAIN_BUILD}/etc" reqdir=${sysconfdir}/pip export PATH="${prefix}/bin:${PATH}" +export PKG_CONFIG_PATH="${prefix}/lib/pkgconfig" export CFLAGS="-I${prefix}/include" export LDFLAGS="-L${prefix}/lib -Wl,-rpath=${prefix}/lib" @@ -24,7 +25,12 @@ function build() # Iterate over pip requirement files for req in ${reqdir}/* do - pip install --upgrade --progress-bar=off -v -r "${req}" + pip install --upgrade --progress-bar=off -r "${req}" + retval=$? + if [[ ${retval} != 0 ]]; then + echo "BUILD FAILED: ${req}" + exit ${retval} + fi done post } @@ -32,8 +38,8 @@ function build() function post() { rm -rf ~/.cache/pip - [[ -d src ]] && rm -rf src - [[ -f gmon.out ]] && rm -rf gmon.out + [[ -d src ]] && rm -rf src || true + [[ -f gmon.out ]] && rm -rf gmon.out || true } build diff --git a/etc/tasks/002-python.sh b/etc/tasks/002-python.sh deleted file mode 100755 index d7db12f..0000000 --- a/etc/tasks/002-python.sh +++ /dev/null @@ -1,133 +0,0 @@ -#!/bin/bash -set -e -set -x - -python_version="${PYTHON_VERSION}" -python_basever="${python_version%.*}" - -if [[ ! ${python_version} || ! ${python_basever} ]]; then - echo "Need a python version..." - exit 1 -fi - -python_base_url="https://www.python.org/ftp/python" -python_tarball="Python-${python_version}.tgz" -python_source="${python_tarball%%.tgz}" -python_url="${python_base_url}/${python_version}/${python_tarball}" -prefix="${TOOLCHAIN}" - -dep_table=( - "bzlib.h libbz2.so" - "expat.h libexpat.so" - "ffi.h libffi.so" - "gdbm.h libgdbm.so" - "lzma.h liblzma.so" - "ncurses.h libncurses.so" - "nislib.h libnsl.so" - "readline.h libreadline.so" - "ssl.h libssl.so" - "sqlite3.h libsqlite3.so" - "tcl.h libtcl.so" - "tk.h libtk.so" - "zlib.h libz.so" -) - - -function depcheck() -{ - dep_count=0 - dep_total="${#dep_table[@]}" - - set +x - for _record in "${dep_table[@]}" - do - unset record - read -ra record <<< $_record - - header=$(find /usr/include /usr/lib{,64} -regex ".*\/${record[0]}" 2>/dev/null | head -n 1 || true) - if [[ -n $header ]]; then - dep_count=$((dep_count+1)) - else - echo "Missing header: ${record[0]}" - fi - lib=$(find /usr/lib{,64} -regex ".*\/${record[1]}" 2>/dev/null | head -n 1 || true) - if [[ -n "$lib" ]]; then - dep_count=$((dep_count+1)) - else - echo "Missing library: ${record[1]}" - fi - done - set -x - - if [[ ${dep_count} != $(( (dep_total * 2) )) ]]; then - echo 'Missing dependencies...' - exit 1 - fi -} - - -function pre() -{ - depcheck - - if [[ ! -f ${python_tarball} ]]; then - curl -LO "${python_url}" - fi - - if [[ -d ${python_source} ]]; then - rm -rf "${python_source}" - fi - - tar xf "${python_tarball}" -} - - -function build() -{ - pre - export CFLAGS="-I${prefix}/include" - export LDFLAGS="-L${prefix}/lib -Wl,-rpath=${prefix}/lib" - pushd "${python_source}" - #--enable-optimizations \ - ./configure \ - --prefix="${prefix}" \ - --enable-ipv6 \ - --enable-loadable-sqlite-extensions \ - --enable-profiling \ - --enable-shared \ - --with-dbmliborder=gdbm:ndbm \ - --with-pymalloc \ - --with-system-expat - make -j4 - make install - popd - post -} - - -function post() -{ - export PATH=$prefix/bin:$PATH - ln -sf python3 "${prefix}"/bin/python - ln -sf python3-config "${prefix}"/bin/python-config - ln -sf idle3 "${prefix}"/bin/idle - ln -sf pydoc3 "${prefix}"/bin/pydoc - ln -sf pip3 "${prefix}"/bin/pip - ln -sf python${python_basever}.1 "${prefix}"/share/man/man1/python.1 - - echo '---' - python --version - python -c "import sys; from pprint import pprint; pprint(sys.path)" - echo '---' - ldd $(which python) - echo '---' - - rm -rf $HOME/.config/pip - rm -rf "${python_tarball}" - rm -rf "${python_source}" - echo "All done." -} - - -# Main -build diff --git a/etc/tasks/003-python-packages.sh b/etc/tasks/003-python-packages.sh deleted file mode 100755 index cb248ff..0000000 --- a/etc/tasks/003-python-packages.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -set -x - -# Uses GLOBAL environment variable: PYTHON_VERSION defined by `docker build` argument -prefix="${TOOLCHAIN}" -sysconfdir="${TOOLCHAIN_BUILD}/etc" -reqdir=${sysconfdir}/pip - -export PATH="${prefix}/bin:${PATH}" -export CFLAGS="-I${prefix}/include" -export LDFLAGS="-L${prefix}/lib -Wl,-rpath=${prefix}/lib" - -function pre() -{ - if [[ ! -d ${reqdir} ]]; then - # Nothing there, but maybe that's on purpose. - exit 0 - fi -} - -function build() -{ - pre - # Iterate over pip requirement files - for req in ${reqdir}/* - do - pip install --upgrade --progress-bar=off -v -r "${req}" - done - post -} - -function post() -{ - rm -rf ~/.cache/pip - [[ -d src ]] && rm -rf src - [[ -f gmon.out ]] && rm -rf gmon.out -} - -build diff --git a/etc/tasks/004-packages.sh b/etc/tasks/004-packages.sh deleted file mode 100755 index f7d0cad..0000000 --- a/etc/tasks/004-packages.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -set -e -set -x - -prefix="${TOOLCHAIN}" -sysconfdir="${TOOLCHAIN_BUILD}/etc" -reqdir=${sysconfdir}/pkgs -blddir=builds - -export PATH="${prefix}/bin:${PATH}" -export CFLAGS="-I${prefix}/include" -export LDFLAGS="-L${prefix}/lib -Wl,-rpath=${prefix}/lib" -export PREFIX="${prefix}" - -function pre() -{ - if [[ ! -d ${reqdir} ]]; then - # Nothing there, but maybe that's on purpose. - exit 0 - fi - mkdir -p "${blddir}" - pushd ${blddir} &>/dev/null -} - -function build() -{ - pre - # Iterate over binary package build scripts - for req in ${reqdir}/* - do - chmod +x "${req}" - "${req}" - done - post -} - -function post() -{ - popd &>/dev/null - [[ -d ${blddir} ]] && rm -rf "${blddir}" -} - -build diff --git a/etc/tasks/998-debase.sh b/etc/tasks/998-debase.sh deleted file mode 100755 index 8b56cd4..0000000 --- a/etc/tasks/998-debase.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -packages=( - gcc - gcc-c++ - gcc-gfortran -) -yum remove -y "${packages[@]}" diff --git a/etc/tasks/999-clean.sh b/etc/tasks/999-clean.sh index 44f5d16..c791ec5 100755 --- a/etc/tasks/999-clean.sh +++ b/etc/tasks/999-clean.sh @@ -4,8 +4,17 @@ if [[ ! -f /.dockerenv ]]; then exit 1 fi +packages=( + gcc + gcc-c++ + gcc-gfortran +) +sudo yum remove -y "${packages[@]}" sudo yum clean all +# Remove all static libraries +sudo find "${TOOLCHAIN_LIB}" -name '*.a' -delete + sudo rm -rf "${HOME}/.astropy" sudo rm -rf "${HOME}"/* sudo rm -rf /tmp/* diff --git a/scripts/build.sh b/scripts/build.sh index 23d0ea1..f7c7a2d 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -1,6 +1,7 @@ #!/bin/bash -e taskdir=${TOOLCHAIN_BUILD}/etc/tasks +export _maxjobs=$(getconf _NPROCESSORS_ONLN) if [[ ! -d ${taskdir} ]]; then echo "No tasks. ${taskdir} does not exist." @@ -16,4 +17,9 @@ do fi echo "Executing: ${task}" ${task} + retval=$? + if [[ ${retval} != 0 ]]; then + echo "TASK FAILED: ${task}" + exit ${retval} + fi done -- cgit