diff options
| -rw-r--r-- | README.md | 6 | ||||
| -rw-r--r-- | jenkins/astroconda_parameterized.groovy | 214 | ||||
| -rw-r--r-- | manifests/dev.yaml | 88 | ||||
| -rw-r--r-- | manifests/etc.yaml | 25 | ||||
| -rw-r--r-- | manifests/public.yaml | 87 | ||||
| -rw-r--r-- | manifests/test.yaml | 8 |
6 files changed, 428 insertions, 0 deletions
diff --git a/README.md b/README.md new file mode 100644 index 0000000..4cb3c6b --- /dev/null +++ b/README.md @@ -0,0 +1,6 @@ +This project contains several files to partially define and facilitate an +automated conda package building system for the astroconda project. + +- `/jenkins` : Jenkins CI pipeline build script(s) ([Groovy](http://www.groovy-lang.org/)) +- `/manifests` : Lists of conda package & metapackage recipes, and associated metadata for various astroconda release types ([YAML](http://yaml.org/)) + diff --git a/jenkins/astroconda_parameterized.groovy b/jenkins/astroconda_parameterized.groovy new file mode 100644 index 0000000..24c8605 --- /dev/null +++ b/jenkins/astroconda_parameterized.groovy @@ -0,0 +1,214 @@ +// Astroconda build control script for use with the Jenkins CI system +//---------------------------------------------------------------------------- +// This script is typically dispatched from a multi-configuration (build +// matrix) job. It accepts the following parameters controlling its behavior +// at dispatch time: +// +// label - The build node on which to execute +// ("nott", "boyle", etc) +// release_type - Determines which manifest to use +// ("dev", "public", "test", etc) +// py_version - The version of python to build upon (ships with conda) +// ("py2.7", "py3.5", ...) +// +// Constants controlling the behavior of this script: +// +// Where to obtain this file and the manifest files +this.build_control_URL = "https://github.com/rendinam/build_control" + +// The conda version shown in the conda_installers list below is installed +// first, then the version is forced to this value. +this.conda_version = "4.2.15" + +// Conda-build is installed fresh at this version. +this.conda_build_version = "2.1.1" + +// Where to get the conda installer +this.conda_base_URL = "https://repo.continuum.io/miniconda/" + +// The conda installer script to use for various <OS><py_version> combinations. +this.conda_installers = ["Linux-py2.7":"Miniconda2-4.2.12-Linux-x86_64.sh", + "Linux-py3.5":"Miniconda3-4.2.12-Linux-x86_64.sh", + "MacOSX-py2.7":"Miniconda2-4.2.12-MacOSX-x86_64.sh", + "MacOSX-py3.5":"Miniconda3-4.2.12-MacOSX-x86_64.sh"] + +// The manifest file to use for each release type +this.manifest_files = ["dev":"dev.yaml", + "public":"public.yaml", + "public_legacy":"public_legacy.yaml", + "ETC":"etc.yaml", + "dev_rotate_meta_packages":"astroconda.dev_rotate.yaml", + "test":"test.yaml"] +//---------------------------------------------------------------------------- + +node(this.label) { + + this.OSname = null + def uname = sh(script: "uname", returnStdout: true).trim() + if (uname == "Darwin") { + this.OSname = "MacOSX" + env.PATH = "${env.PATH}:/sw/bin" + this.CONDA_BLD_OUTPUT_DIR = "osx-64" + } + if (uname == "Linux") { + this.OSname = uname + this.CONDA_BLD_OUTPUT_DIR = "linux-64" + } + assert uname != null + + this.BUILD_SUBDIR = "${this.release_type}/${this.py_version}" + this.BUILD_ROOT = "${env.WORKSPACE}/${this.BUILD_SUBDIR}" + currentBuild.displayName = + "${this.release_type}-${this.OSname}-${this.py_version}-${this.label}" + + // Delete any existing job workspace directory contents. + // The directory deleted is the one named after the jenkins pipeline job. + deleteDir() + + // Perform the following build job tasks in BUILD_SUBDIR within the + // topmost workspace directory for this node to allow for multiple + // parallel jobs using different combinations of build parameters. + dir("${this.BUILD_SUBDIR}") { + + stage('Setup') { + + println "this.label ${this.label}" + assert this.label != null + assert this.label != "label-DEFAULT" + + println "this.py_version ${this.py_version}" + assert this.py_version != null + assert this.py_version != "py_version-DEFAULT" + + println "this.release_type ${this.release_type}" + assert this.release_type != null + assert this.release_type != "release_type-DEFAULT" + + // Fetch the manifest files + git url: this.build_control_URL + + // Check for the availability of a download tool and then use it + // to get the conda installer. + def dl_cmds = ["wget --no-verbose --server-response --no-check-certificate", + "curl -OSs"] + def dl_cmd = null + def stat1 = 999 + for (cmd in dl_cmds) { + stat1 = sh(script: "which ${cmd.split()[0]}", returnStatus: true) + if( stat1 == 0 ) { + dl_cmd = cmd + break + } + } + if (stat1 != 0) { + println("Could not find a download tool. Unable to proceed.") + sh "false" + } + + def conda_installer = + this.conda_installers["${this.OSname}-${this.py_version}"] + dl_cmd = dl_cmd + " ${this.conda_base_URL}${conda_installer}" + sh dl_cmd + + // Make the log files a bit more deterministic + env.PYTHONUNBUFFERED = "true" + + // Run miniconda installer and then force to particular version + sh "bash ./${conda_installer} -b -p miniconda" + env.PATH = "${this.BUILD_ROOT}/miniconda/bin/:" + "${env.PATH}" + sh "conda install --quiet conda=${this.conda_version}" + sh "conda install --quiet --yes conda-build=${this.conda_build_version}" + + // TODO: Check for presence of build support tools, such as git, + // make, compilers, etc. and print a summary of their versions + // for auditing purposes. + + this.manifest = readYaml file: "manifests/" + + this.manifest_files["${this.release_type}"] + println("Manifest repository: ${this.manifest.repository}") + println("Manifest numpy version specification: " + + "${this.manifest.numpy_version}") + println("Manifest packages to build:") + for (pkgname in this.manifest.packages) { + println(pkgname) + } + + println("Checking for supplemental channels specification:") + if (this.manifest.channels != null) { + println("Supplemental channel(s) found, adding:") + for (channel in this.manifest.channels) { + sh "conda config --add channels ${channel}" + } + } else { + println("[INFO] No supplemental channels specified in manifest.") + } + + // Retrieve conda recipes + def recipes_dir = "conda-recipes" + dir(recipes_dir) { + git url: this.manifest.repository + } + + // Set git ID info to avoid errors with commands like 'git tag'. + sh "git config user.email \"ssb-jenkins@ssb-jenkins\"" + sh "git config user.name \"ssb-jenkins\"" + } + + stage('Build packages') { + def build_cmd = "conda build" + def pyversion_num = this.py_version.split('y')[1] + def build_args = "--no-anaconda-upload --python=${pyversion_num}" + + " --numpy=${this.manifest.numpy_version} --skip-existing" + def stat2 = 999 + dir ("conda-recipes") { + for (pkgname in this.manifest.packages) { + stat2 = sh(script: "${build_cmd} ${build_args} ${pkgname}", + returnStatus: true) + println("Shell call returned status: ${stat2}") + println "\nEnd Build: ${pkgname} " + + "=======================================================" + } + } + + // Determine if each package in the manifest was built. + // Set flag to fail the stage in the event of any missing packages. + def stage_success = "true" + def outdir = "miniconda/conda-bld/${this.CONDA_BLD_OUTPUT_DIR}" + dir(outdir) { + def built_pkgs = sh(script: "ls -1 *.tar.bz2", + returnStdout: true).trim().split() + def found_pkg = false + def first = true + for (pkgname in this.manifest.packages) { + found_pkg = false + for (built_pkg in built_pkgs) { + if ( built_pkg.indexOf(pkgname, 0) != -1) { + found_pkg = true + break + } + } + if (!found_pkg) { + if (first) { + println("ERROR BUILDING ONE OR MORE PACKAGES!\n") + first = false + } + println("ERROR building: ${pkgname}") + stage_success = "false" + } + } + } + sh stage_success + } + + stage('Tests') { + println "Test results go here." + } + + stage('Publish/Archive build products') { + // Publishing and/or archival steps go here. + } + + } // end dir() + +} //end node + diff --git a/manifests/dev.yaml b/manifests/dev.yaml new file mode 100644 index 0000000..2992391 --- /dev/null +++ b/manifests/dev.yaml @@ -0,0 +1,88 @@ +repository: 'https://github.com/astroconda/astroconda-dev' +numpy_version: 1.11 +packages: + - stsci.tools + - stsci.imagestats + - stsci-data-analysis + - stsci-hst + - stsci + + - verhawk + - acstools + - appdirs + - aprio + - asdf-standard + - asdf + - astroimtools + - astropy + - asv + - calcos + - cfitsio + - costools + - crds + - cube-tools + - d2to1 + - drizzle + - drizzlepac + - ds9 + - fftw + - fitsblender + - ginga + - glueviz + - glue-vispy-viewers + - gwcs + - hstcal + - htc_utils + - imexam + - jplephem + - jwst + - jwst_coronagraph_visibility + - jwst_gtvt + - jwxml + - mosviz + - nictools + - opuscoords +# - pandokia + - photutils + - poppy + - purge_path + - pyds9 + - pyfftw + - pyqtgraph + - pyraf + - pyregion + - pysynphot + - python-daemon + - pytools + - pywcs + - reftools + - relic + - selenium + - shunit2 + - specviz + - specutils + - sphere + - sphinx_rtd_theme + - sphinxcontrib-programoutput + - stginga + - stistools + - stsci.convolve + - stsci.distutils + - stsci.image + - stsci.imagemanip + - stsci.ndimage + - stsci.numdisplay + - stsci.skypac + - stsci.sphere + - stsci.sphinxext + - stsci.stimage + - stwcs + - synphot + - stsynphot + - wcstools + - webbpsf-data + - webbpsf + - wfc3tools + - wfpc2tools + - xpa + diff --git a/manifests/etc.yaml b/manifests/etc.yaml new file mode 100644 index 0000000..7a4cc09 --- /dev/null +++ b/manifests/etc.yaml @@ -0,0 +1,25 @@ +repository: 'https://github.com/astroconda/astroconda-etc' +channels: + - 'http://ssb.stsci.edu/astroconda' +numpy_version: 1.11 +packages: + - pandeia-thirdparty + - pyetc-thirdparty + - ext_shmht + - django + - epydoc + - linecache2 + - mechanize + - parsley + - pmw + - pyfits + - sphinx-bootstrap-theme + - sphinx_rtd_theme + - threading2 + - traceback2 + - urwid + - websocket-client + - ws4py + - wsgiref + - yolk + diff --git a/manifests/public.yaml b/manifests/public.yaml new file mode 100644 index 0000000..c7b7761 --- /dev/null +++ b/manifests/public.yaml @@ -0,0 +1,87 @@ +repository: 'https://github.com/astroconda/astroconda-contrib' +numpy_version: 1.11 +packages: + - stsci.tools + - stsci.imagestats + - stsci-data-analysis + - stsci-hst + - stsci + + - verhawk + - acstools + - appdirs + - aprio + - asdf-standard + - asdf + - astroimtools + - asv + - calcos + - cfitsio + - costools + - crds + - d2to1 + - drizzle + - drizzlepac + - ds9 + - fftw + - fitsblender + - fitsverify + - ginga + - glueviz + - glue-vispy-viewers + - glue-ginga + - gwcs + - hstcal + - htc_utils + - imexam + - jwst_coronagraph_visibility + - jwst_gtvt + - jwxml + - mosviz + - nictools + - opuscoords +# - pandokia + - photutils + - poppy + - purge_path + - pyds9 + - pyfftw + - pyqtgraph + - pyraf + - pyregion + - pysynphot + - python-daemon + - pytools + - pywcs + - reftools + - relic + - selenium + - shunit2 + - specviz + - specutils + - sphere + - sphinx_rtd_theme + - stsci_sphinx_theme + - sphinxcontrib-programoutput + - stginga + - stistools + - stsci.convolve + - stsci.distutils + - stsci.image + - stsci.imagemanip + - stsci.ndimage + - stsci.numdisplay + - stsci.skypac + - stsci.sphere + - stsci.sphinxext + - stsci.stimage + - stwcs + - synphot + - stsynphot + - wcstools + - webbpsf-data + - webbpsf + - wfc3tools + - wfpc2tools + - xpa + diff --git a/manifests/test.yaml b/manifests/test.yaml new file mode 100644 index 0000000..4f0ff2d --- /dev/null +++ b/manifests/test.yaml @@ -0,0 +1,8 @@ +repository: 'https://github.com/rendinam/astroconda-contrib' +numpy_version: 1.11 +packages: + - verhawk +# - costools + - INTENTIONALLY_INVALID +# - acstools + |