diff options
| -rw-r--r-- | jenkins/dispatch.groovy | 23 | ||||
| -rw-r--r-- | jenkins/generator_DSL.groovy | 1 | ||||
| -rw-r--r-- | jenkins/obsolete/astroconda_parameterized.groovy | 212 | ||||
| -rw-r--r-- | jenkins/package_builder.groovy | 41 | ||||
| -rw-r--r-- | jenkins/version_pins.yml | 13 | ||||
| -rw-r--r-- | manifests/dev-test.yaml | 2 | ||||
| -rw-r--r-- | manifests/public-test.yaml | 10 |
7 files changed, 76 insertions, 226 deletions
diff --git a/jenkins/dispatch.groovy b/jenkins/dispatch.groovy index 80cfe89..a1e91cd 100644 --- a/jenkins/dispatch.groovy +++ b/jenkins/dispatch.groovy @@ -27,6 +27,10 @@ this.conda_installers = ["Linux-py2":"Miniconda2-${CONDA_VERSION}-Linux-x86_64. this.max_publication_tries = 5 this.publication_lock_wait_s = 10 +// Name of YAML file that contains global pinning information to use during the build. +// Packages that appear in this file will be pinned to the version indicated. +this.version_pins_file = "version_pins.yml" + node(LABEL) { this.OSname = null @@ -76,6 +80,8 @@ node(LABEL) { this.manifest.channel_URL = this.manifest.channel_URL[0..-2] } + this.pins_file = readYaml file: "jenkins/${this.version_pins_file}" + // Allow for sharing build_list between stages below. this.build_list = [] @@ -195,6 +201,23 @@ node(LABEL) { sh "patch ${filename} ${full_patchname}" } + // (conda-build 3.x only) + // Create and populate environment to be used for pinning reference when + // building packages via the --bootstrap flag. + // sh "conda create --name pin_env python=${PY_VERSION}" + if (CONDA_BUILD_VERSION[0] == "3") { + println("Creating environment based on package pin values found \n" + + "in ${this.version_pins_file} to use as global version pinnning \n" + + "specification.") + def env_cmd = "conda create --quiet -n pin_env python=${PY_VERSION}" + for (pkg in this.pins_file.packages) { + // TODO: Don't let conda components update here. + env_cmd = "${env_cmd} ${pkg.tokenize()[0]}=${pkg.tokenize()[1]}" + } + sh "${env_cmd}" + sh "source activate pin_env; conda env list; conda list" + } + // Install support tools dir(this.utils_dir) { sh "python setup.py install" diff --git a/jenkins/generator_DSL.groovy b/jenkins/generator_DSL.groovy index 21088c0..6474e9c 100644 --- a/jenkins/generator_DSL.groovy +++ b/jenkins/generator_DSL.groovy @@ -100,6 +100,7 @@ for (label in labels.trim().tokenize()) { pipelineJob("${suite_name}/${pkg}") { environmentVariables { env("JOB_DEF_GENERATION_TIME", job_def_generation_time) + env("CONDA_BUILD_VERSION", conda_build_version) } parameters { stringParam("label", diff --git a/jenkins/obsolete/astroconda_parameterized.groovy b/jenkins/obsolete/astroconda_parameterized.groovy deleted file mode 100644 index 67932ae..0000000 --- a/jenkins/obsolete/astroconda_parameterized.groovy +++ /dev/null @@ -1,212 +0,0 @@ -// Astroconda build control script for use with the Jenkins CI system -//---------------------------------------------------------------------------- -// This script is typically dispatched from a multi-configuration (build -// matrix) job. It accepts the following parameters controlling its behavior -// at dispatch time: -// -// label - The build node on which to execute -// ("nott", "boyle", etc) -// release_type - Determines which manifest to use -// ("dev", "public", "test", etc) -// py_version - The version of python to build upon (ships with conda) -// ("py2.7", "py3.5", ...) -// -// Constants controlling the behavior of this script: -// -// Where to obtain this file and the manifest files -this.build_control_URL = "https://github.com/astroconda/build_control" - -// The conda version shown in the conda_installers list below is installed -// first, then the version is forced to this value. -this.conda_version = "4.2.15" - -// Conda-build is installed fresh at this version. -this.conda_build_version = "2.1.1" - -// Where to get the conda installer -this.conda_base_URL = "https://repo.continuum.io/miniconda/" - -// The conda installer script to use for various <OS><py_version> combinations. -this.conda_installers = ["Linux-py2.7":"Miniconda2-4.2.12-Linux-x86_64.sh", - "Linux-py3.5":"Miniconda3-4.2.12-Linux-x86_64.sh", - "MacOSX-py2.7":"Miniconda2-4.2.12-MacOSX-x86_64.sh", - "MacOSX-py3.5":"Miniconda3-4.2.12-MacOSX-x86_64.sh"] - -// The manifest file to use for each release type -this.manifest_files = ["dev":"dev.yaml", - "public":"public.yaml", - "public_legacy":"public_legacy.yaml", - "ETC":"etc.yaml", - "dev_rotate_meta_packages":"astroconda.dev_rotate.yaml", - "test":"test.yaml"] -//---------------------------------------------------------------------------- - -node(this.label) { - - this.OSname = null - def uname = sh(script: "uname", returnStdout: true).trim() - if (uname == "Darwin") { - this.OSname = "MacOSX" - env.PATH = "${env.PATH}:/sw/bin" - this.CONDA_BLD_OUTPUT_DIR = "osx-64" - } - if (uname == "Linux") { - this.OSname = uname - this.CONDA_BLD_OUTPUT_DIR = "linux-64" - } - assert uname != null - - this.BUILD_SUBDIR = "${this.release_type}/${this.py_version}" - this.BUILD_ROOT = "${env.WORKSPACE}/${this.BUILD_SUBDIR}" - - // Delete any existing job workspace directory contents. - // The directory deleted is the one named after the jenkins pipeline job. - deleteDir() - - // Perform the following build job tasks in BUILD_SUBDIR within the - // topmost workspace directory for this node to allow for multiple - // parallel jobs using different combinations of build parameters. - dir("${this.BUILD_SUBDIR}") { - - stage('Setup') { - - println "this.label ${this.label}" - assert this.label != null - assert this.label != "label-DEFAULT" - - println "this.py_version ${this.py_version}" - assert this.py_version != null - assert this.py_version != "py_version-DEFAULT" - - println "this.release_type ${this.release_type}" - assert this.release_type != null - assert this.release_type != "release_type-DEFAULT" - - // Fetch the manifest files - git url: this.build_control_URL - - // Check for the availability of a download tool and then use it - // to get the conda installer. - def dl_cmds = ["wget --no-verbose --server-response --no-check-certificate", - "curl -OSs"] - def dl_cmd = null - def stat1 = 999 - for (cmd in dl_cmds) { - stat1 = sh(script: "which ${cmd.split()[0]}", returnStatus: true) - if( stat1 == 0 ) { - dl_cmd = cmd - break - } - } - if (stat1 != 0) { - println("Could not find a download tool. Unable to proceed.") - sh "false" - } - - def conda_installer = - this.conda_installers["${this.OSname}-${this.py_version}"] - dl_cmd = dl_cmd + " ${this.conda_base_URL}${conda_installer}" - sh dl_cmd - - // Make the log files a bit more deterministic - env.PYTHONUNBUFFERED = "true" - - // Run miniconda installer and then force to particular version - sh "bash ./${conda_installer} -b -p miniconda" - env.PATH = "${this.BUILD_ROOT}/miniconda/bin/:" + "${env.PATH}" - sh "conda install --quiet conda=${this.conda_version}" - sh "conda install --quiet --yes conda-build=${this.conda_build_version}" - - // TODO: Check for presence of build support tools, such as git, - // make, compilers, etc. and print a summary of their versions - // for auditing purposes. - - this.manifest = readYaml file: "manifests/" + - this.manifest_files["${this.release_type}"] - println("Manifest repository: ${this.manifest.repository}") - println("Manifest numpy version specification: " + - "${this.manifest.numpy_version}") - println("Manifest packages to build:") - for (pkgname in this.manifest.packages) { - println(pkgname) - } - - println("Checking for supplemental channels specification:") - if (this.manifest.channels != null) { - println("Supplemental channel(s) found, adding:") - for (channel in this.manifest.channels) { - sh "conda config --add channels ${channel}" - } - } else { - println("[INFO] No supplemental channels specified in manifest.") - } - - // Retrieve conda recipes - def recipes_dir = "conda-recipes" - dir(recipes_dir) { - git url: this.manifest.repository - } - - // Set git ID info to avoid errors with commands like 'git tag'. - sh "git config user.email \"ssb-jenkins@ssb-jenkins\"" - sh "git config user.name \"ssb-jenkins\"" - } - - stage('Build packages') { - def build_cmd = "conda build" - def pyversion_num = this.py_version.split('y')[1] - def build_args = "--no-anaconda-upload --python=${pyversion_num}" + - " --numpy=${this.manifest.numpy_version} --skip-existing" - def stat2 = 999 - dir ("conda-recipes") { - for (pkgname in this.manifest.packages) { - stat2 = sh(script: "${build_cmd} ${build_args} ${pkgname}", - returnStatus: true) - println("Shell call returned status: ${stat2}") - println "\nEnd Build: ${pkgname} " + - "=======================================================" - } - } - - // Determine if each package in the manifest was built. - // Set flag to fail the stage in the event of any missing packages. - def stage_success = "true" - def outdir = "miniconda/conda-bld/${this.CONDA_BLD_OUTPUT_DIR}" - dir(outdir) { - def built_pkgs = sh(script: "ls -1 *.tar.bz2", - returnStdout: true).trim().split() - def found_pkg = false - def first = true - for (pkgname in this.manifest.packages) { - found_pkg = false - for (built_pkg in built_pkgs) { - if ( built_pkg.indexOf(pkgname, 0) != -1) { - found_pkg = true - break - } - } - if (!found_pkg) { - if (first) { - println("ERROR BUILDING ONE OR MORE PACKAGES!\n") - first = false - } - println("ERROR building: ${pkgname}") - stage_success = "false" - } - } - } - sh stage_success - } - - stage('Tests') { - println "Test results go here." - } - - stage('Publish/Archive build products') { - // Publishing and/or archival steps go here. - } - - } // end dir() - -} //end node - diff --git a/jenkins/package_builder.groovy b/jenkins/package_builder.groovy index a63c1b8..b7ad786 100644 --- a/jenkins/package_builder.groovy +++ b/jenkins/package_builder.groovy @@ -1,3 +1,7 @@ +// Parameters inherited environment injection. +//---------------------------------------------------------------------------- +// CONDA_BUILD_VERSION - Conda-build is installed forced to this version. + this.build_status_file = "${this.parent_workspace}/propagated_build_status" node(this.label) { @@ -45,17 +49,7 @@ node(this.label) { cmd = "conda build" - // Use channel URL obtained from manifest in build command if - // manifest has been culled to allow packages being built to - // simply download dependency packages from the publication - // channel as needed, rather than build them as part of the - // package build session that requires them. - def channel_option = "--channel ${this.channel_URL}" - stage("Build") { - if (this.cull_manifest == "false") { - channel_option = "" - } build_cmd = cmd args = ["--no-test", "--no-anaconda-upload", @@ -64,8 +58,24 @@ node(this.label) { "--skip-existing", "--override-channels", "--channel defaults", - "${channel_option}", "--dirty"] + // Use channel URL obtained from manifest in build command if + // manifest has been culled to allow packages being built to + // simply download dependency packages from the publication + // channel as needed, rather than build them as part of the + // package build session that requires them. + if (this.cull_manifest == "true") { + args.add("--channel ${this.channel_URL}") + } + // If conda build 3.x is being used, apply any global package + // pin values contained in the 'pin_env' conda environment + // created by the dispatch job by using the --bootstrap flag + // here. + if (CONDA_BUILD_VERSION[0] == "3") { + args.add("--old-build-string") + args.add("--bootstrap pin_env") + } + // Compose build command string to use in shell call. for (arg in args) { build_cmd = "${build_cmd} ${arg}" } @@ -90,8 +100,13 @@ node(this.label) { "--python=${this.py_version}", "--numpy=${this.numpy_version}", "--override-channels", - "--channel defaults", - "${channel_option}"] + "--channel defaults"] + if (this.cull_manifest == "true") { + args.add("--channel ${this.channel_URL}") + } + if (CONDA_BUILD_VERSION[0] == "3") { + args.add("--old-build-string") + } for (arg in args) { build_cmd = "${build_cmd} ${arg}" } diff --git a/jenkins/version_pins.yml b/jenkins/version_pins.yml new file mode 100644 index 0000000..caaa16a --- /dev/null +++ b/jenkins/version_pins.yml @@ -0,0 +1,13 @@ +# This file is only honored when the build system is configured to use +# conda-build 3.x. +# +# It allows the definition of a list of core dependency packages to pin to +# the supplied values when building packages from a manifest in a job suite. +# +# Example use case: +# - setuptools 36.4.0 is released in a broken state which prevents packages +# that have it as a build requirement from building. +# - setuptools can be pinned to an earlier version for all package builds +# that happen in a given job suite, allowing them to succeed. +packages: + - setuptools 27.2.0 diff --git a/manifests/dev-test.yaml b/manifests/dev-test.yaml index 531bf80..68da460 100644 --- a/manifests/dev-test.yaml +++ b/manifests/dev-test.yaml @@ -7,7 +7,7 @@ channel_URL: 'http://ssb.stsci.edu/astroconda-dev' # publication_root path needs to be visible from the slave nodes. publication_root: '/eng/ssb/websites/ssbpublic/astroconda-j-dev-testing' packages: - - fitsverify + - drizzlepac # - crds # - cube-tools # - sphinxcontrib-programoutput diff --git a/manifests/public-test.yaml b/manifests/public-test.yaml new file mode 100644 index 0000000..331d857 --- /dev/null +++ b/manifests/public-test.yaml @@ -0,0 +1,10 @@ +# Recipe repository +repository: 'https://github.com/astroconda/astroconda-contrib' + +# Publication channel to consult when determining what packages already exist. +channel_URL: 'http://ssb.stsci.edu/astroconda' + +# publication_root path needs to be visible from the slave nodes. +publication_root: '/eng/ssb/websites/ssbpublic/astroconda-j-pub-staging' +packages: + - asdf |