Merge pull request #31 from rendinam/pkg_pinning0.2.3

Allow pinning build support packages to requested versions...

7 files changed, 76 insertions, 226 deletions

diff --git a/jenkins/dispatch.groovy b/jenkins/dispatch.groovy
index 80cfe89..a1e91cd 100644
--- a/jenkins/dispatch.groovy
+++ b/jenkins/dispatch.groovy
@@ -27,6 +27,10 @@ this.conda_installers  = ["Linux-py2":"Miniconda2-${CONDA_VERSION}-Linux-x86_64.
 this.max_publication_tries = 5
 this.publication_lock_wait_s = 10
 
+// Name of YAML file that contains global pinning information to use during the build.
+// Packages that appear in this file will be pinned to the version indicated.
+this.version_pins_file = "version_pins.yml"
+
 node(LABEL) {
 
     this.OSname = null
@@ -76,6 +80,8 @@ node(LABEL) {
         this.manifest.channel_URL = this.manifest.channel_URL[0..-2]
     }
 
+    this.pins_file = readYaml file: "jenkins/${this.version_pins_file}"
+
     // Allow for sharing build_list between stages below.
     this.build_list = []
 
@@ -195,6 +201,23 @@ node(LABEL) {
             sh "patch ${filename} ${full_patchname}"
         }
 
+        // (conda-build 3.x only)
+        // Create and populate environment to be used for pinning reference when
+        // building packages via the --bootstrap flag.
+        // sh "conda create --name pin_env python=${PY_VERSION}"
+        if (CONDA_BUILD_VERSION[0] == "3") {
+            println("Creating environment based on package pin values found \n" +
+            "in ${this.version_pins_file} to use as global version pinnning \n" +
+            "specification.")
+            def env_cmd = "conda create --quiet -n pin_env python=${PY_VERSION}"
+            for (pkg in this.pins_file.packages) {
+                // TODO: Don't let conda components update here.
+                env_cmd = "${env_cmd} ${pkg.tokenize()[0]}=${pkg.tokenize()[1]}"
+            }
+            sh "${env_cmd}"
+            sh "source activate pin_env; conda env list; conda list"
+        }
+
         // Install support tools
         dir(this.utils_dir) {
             sh "python setup.py install"
diff --git a/jenkins/generator_DSL.groovy b/jenkins/generator_DSL.groovy
index 21088c0..6474e9c 100644
--- a/jenkins/generator_DSL.groovy
+++ b/jenkins/generator_DSL.groovy
@@ -100,6 +100,7 @@ for (label in labels.trim().tokenize()) {
                 pipelineJob("${suite_name}/${pkg}") {
                     environmentVariables {
                         env("JOB_DEF_GENERATION_TIME", job_def_generation_time)
+                        env("CONDA_BUILD_VERSION", conda_build_version)
                     }
                     parameters {
                         stringParam("label",
diff --git a/jenkins/obsolete/astroconda_parameterized.groovy b/jenkins/obsolete/astroconda_parameterized.groovy
deleted file mode 100644
index 67932ae..0000000
--- a/jenkins/obsolete/astroconda_parameterized.groovy
+++ /dev/null
@@ -1,212 +0,0 @@
-// Astroconda build control script for use with the Jenkins CI system
-//----------------------------------------------------------------------------
-// This script is typically dispatched from a multi-configuration (build
-// matrix) job.  It accepts the following parameters controlling its behavior
-// at dispatch time:
-//
-//        label - The build node on which to execute
-//                ("nott", "boyle", etc)
-// release_type - Determines which manifest to use
-//                ("dev", "public", "test", etc)
-//   py_version - The version of python to build upon (ships with conda)
-//                ("py2.7", "py3.5", ...)
-// 
-//      Constants controlling the behavior of this script:
-//
-// Where to obtain this file and the manifest files
-this.build_control_URL = "https://github.com/astroconda/build_control"
-
-// The conda version shown in the conda_installers list below is installed
-// first, then the version is forced to this value. 
-this.conda_version = "4.2.15"
-
-// Conda-build is installed fresh at this version.
-this.conda_build_version = "2.1.1"
-
-// Where to get the conda installer
-this.conda_base_URL = "https://repo.continuum.io/miniconda/"
-
-// The conda installer script to use for various <OS><py_version> combinations.
-this.conda_installers  = ["Linux-py2.7":"Miniconda2-4.2.12-Linux-x86_64.sh",
-                          "Linux-py3.5":"Miniconda3-4.2.12-Linux-x86_64.sh",
-                          "MacOSX-py2.7":"Miniconda2-4.2.12-MacOSX-x86_64.sh",
-                          "MacOSX-py3.5":"Miniconda3-4.2.12-MacOSX-x86_64.sh"]
-
-// The manifest file to use for each release type
-this.manifest_files = ["dev":"dev.yaml",
-                       "public":"public.yaml",
-                       "public_legacy":"public_legacy.yaml",
-                       "ETC":"etc.yaml",
-                       "dev_rotate_meta_packages":"astroconda.dev_rotate.yaml",
-                       "test":"test.yaml"]
-//----------------------------------------------------------------------------
-
-node(this.label) {
-
-    this.OSname = null
-    def uname = sh(script: "uname", returnStdout: true).trim()
-    if (uname == "Darwin") {
-        this.OSname = "MacOSX"
-        env.PATH = "${env.PATH}:/sw/bin"
-        this.CONDA_BLD_OUTPUT_DIR = "osx-64"
-    }
-    if (uname == "Linux") {
-        this.OSname = uname
-        this.CONDA_BLD_OUTPUT_DIR = "linux-64"
-    }
-    assert uname != null
-
-    this.BUILD_SUBDIR = "${this.release_type}/${this.py_version}"
-    this.BUILD_ROOT = "${env.WORKSPACE}/${this.BUILD_SUBDIR}"
-
-    // Delete any existing job workspace directory contents.
-    // The directory deleted is the one named after the jenkins pipeline job.
-    deleteDir()
-   
-    // Perform the following build job tasks in BUILD_SUBDIR within the
-    // topmost workspace directory for this node to allow for multiple
-    // parallel jobs using different combinations of build parameters.
-    dir("${this.BUILD_SUBDIR}") {
-
-        stage('Setup') {
-
-            println "this.label ${this.label}"
-            assert this.label != null
-            assert this.label != "label-DEFAULT"
-    
-            println "this.py_version ${this.py_version}"
-            assert this.py_version != null
-            assert this.py_version != "py_version-DEFAULT"
-    
-            println "this.release_type ${this.release_type}"
-            assert this.release_type != null
-            assert this.release_type != "release_type-DEFAULT"
-
-            // Fetch the manifest files
-            git url: this.build_control_URL
-
-            // Check for the availability of a download tool and then use it
-            // to get the conda installer.
-            def dl_cmds = ["wget --no-verbose --server-response --no-check-certificate",
-                           "curl -OSs"]
-            def dl_cmd = null
-            def stat1 = 999
-            for (cmd in dl_cmds) {
-                stat1 = sh(script: "which ${cmd.split()[0]}", returnStatus: true)
-                if( stat1 == 0 ) {
-                    dl_cmd = cmd
-                    break
-                }
-            }
-            if (stat1 != 0) {
-                println("Could not find a download tool. Unable to proceed.")
-                sh "false"
-            }
-
-            def conda_installer =
-               this.conda_installers["${this.OSname}-${this.py_version}"]
-            dl_cmd = dl_cmd + " ${this.conda_base_URL}${conda_installer}"
-            sh dl_cmd
-
-            // Make the log files a bit more deterministic
-            env.PYTHONUNBUFFERED = "true"
-
-            // Run miniconda installer and then force to particular version
-            sh "bash ./${conda_installer} -b -p miniconda"
-            env.PATH = "${this.BUILD_ROOT}/miniconda/bin/:" + "${env.PATH}"
-            sh "conda install --quiet conda=${this.conda_version}"
-            sh "conda install --quiet --yes conda-build=${this.conda_build_version}"
-
-            // TODO: Check for presence of build support tools, such as git,
-            // make, compilers, etc. and print a summary of their versions
-            // for auditing purposes.
-
-            this.manifest = readYaml file: "manifests/" +
-                this.manifest_files["${this.release_type}"]
-            println("Manifest repository: ${this.manifest.repository}")
-            println("Manifest numpy version specification: " +
-                "${this.manifest.numpy_version}")
-            println("Manifest packages to build:")
-            for (pkgname in this.manifest.packages) {
-                println(pkgname)
-            }
-
-            println("Checking for supplemental channels specification:")
-            if (this.manifest.channels != null) {
-                println("Supplemental channel(s) found, adding:")
-                for (channel in this.manifest.channels) {
-                    sh "conda config --add channels ${channel}"
-                }
-            } else {
-                println("[INFO] No supplemental channels specified in manifest.")
-            } 
-
-            // Retrieve conda recipes
-            def recipes_dir = "conda-recipes"
-            dir(recipes_dir) {
-                git url: this.manifest.repository
-            }
-
-            // Set git ID info to avoid errors with commands like 'git tag'.
-            sh "git config user.email \"ssb-jenkins@ssb-jenkins\""
-            sh "git config user.name \"ssb-jenkins\""
-        }
-
-        stage('Build packages') {
-            def build_cmd = "conda build"
-            def pyversion_num = this.py_version.split('y')[1]
-            def build_args = "--no-anaconda-upload --python=${pyversion_num}" +
-                " --numpy=${this.manifest.numpy_version} --skip-existing"
-            def stat2 = 999
-            dir ("conda-recipes") {
-                for (pkgname in this.manifest.packages) {
-                    stat2 = sh(script: "${build_cmd} ${build_args} ${pkgname}",
-                                returnStatus: true)
-                    println("Shell call returned status: ${stat2}")
-                    println "\nEnd Build: ${pkgname} " +
-                        "======================================================="
-                }
-            }
-
-            // Determine if each package in the manifest was built.
-            // Set flag to fail the stage in the event of any missing packages.
-            def stage_success = "true"
-            def outdir = "miniconda/conda-bld/${this.CONDA_BLD_OUTPUT_DIR}"
-            dir(outdir) {
-                def built_pkgs = sh(script: "ls -1 *.tar.bz2",
-                                    returnStdout: true).trim().split()
-                def found_pkg = false
-                def first = true
-                for (pkgname in this.manifest.packages) {
-                    found_pkg = false
-                    for (built_pkg in built_pkgs) {
-                        if ( built_pkg.indexOf(pkgname, 0) != -1) {
-                            found_pkg = true
-                            break
-                        }
-                    }
-                    if (!found_pkg) {
-                        if (first) {
-                            println("ERROR BUILDING ONE OR MORE PACKAGES!\n")
-                            first = false
-                        }
-                        println("ERROR building: ${pkgname}")
-                        stage_success = "false"
-                    }
-                }
-            }
-            sh stage_success
-        }
-
-        stage('Tests') {
-            println "Test results go here."
-        }
-
-        stage('Publish/Archive build products') {
-            // Publishing and/or archival steps go here.
-        }
-
-    } // end dir()
-
-} //end node
-
diff --git a/jenkins/package_builder.groovy b/jenkins/package_builder.groovy
index a63c1b8..b7ad786 100644
--- a/jenkins/package_builder.groovy
+++ b/jenkins/package_builder.groovy
@@ -1,3 +1,7 @@
+// Parameters inherited environment injection.
+//----------------------------------------------------------------------------
+// CONDA_BUILD_VERSION  - Conda-build is installed forced to this version.
+
 this.build_status_file = "${this.parent_workspace}/propagated_build_status"
 
 node(this.label) {
@@ -45,17 +49,7 @@ node(this.label) {
 
             cmd = "conda build"
 
-            // Use channel URL obtained from manifest in build command if
-            // manifest has been culled to allow packages being built to
-            // simply download dependency packages from the publication
-            // channel as needed, rather than build them as part of the
-            // package build session that requires them.
-            def channel_option = "--channel ${this.channel_URL}"
-
             stage("Build") {
-                if (this.cull_manifest == "false") {
-                    channel_option = ""
-                }
                 build_cmd = cmd
                 args = ["--no-test",
                         "--no-anaconda-upload",
@@ -64,8 +58,24 @@ node(this.label) {
                         "--skip-existing",
                         "--override-channels",
                         "--channel defaults",
-                        "${channel_option}",
                         "--dirty"]
+                // Use channel URL obtained from manifest in build command if
+                // manifest has been culled to allow packages being built to
+                // simply download dependency packages from the publication
+                // channel as needed, rather than build them as part of the
+                // package build session that requires them.
+                if (this.cull_manifest == "true") {
+                    args.add("--channel ${this.channel_URL}")
+                }
+                // If conda build 3.x is being used, apply any global package
+                // pin values contained in the 'pin_env' conda environment
+                // created by the dispatch job by using the --bootstrap flag
+                // here.
+                if (CONDA_BUILD_VERSION[0] == "3") {
+                    args.add("--old-build-string")
+                    args.add("--bootstrap pin_env")
+                }
+                // Compose build command string to use in shell call.
                 for (arg in args) {
                     build_cmd = "${build_cmd} ${arg}"
                 }
@@ -90,8 +100,13 @@ node(this.label) {
                             "--python=${this.py_version}",
                             "--numpy=${this.numpy_version}",
                             "--override-channels",
-                            "--channel defaults",
-                            "${channel_option}"]
+                            "--channel defaults"]
+                    if (this.cull_manifest == "true") {
+                        args.add("--channel ${this.channel_URL}")
+                    }
+                    if (CONDA_BUILD_VERSION[0] == "3") {
+                        args.add("--old-build-string")
+                    }
                     for (arg in args) {
                         build_cmd = "${build_cmd} ${arg}"
                     }
diff --git a/jenkins/version_pins.yml b/jenkins/version_pins.yml
new file mode 100644
index 0000000..caaa16a
--- /dev/null
+++ b/jenkins/version_pins.yml
@@ -0,0 +1,13 @@
+# This file is only honored when the build system is configured to use
+# conda-build 3.x.
+#
+# It allows the definition of a list of core dependency packages to pin to
+# the supplied values when building packages from a manifest in a job suite.
+#
+# Example use case:
+#   - setuptools 36.4.0 is released in a broken state which prevents packages
+#     that have it as a build requirement from building.
+#   - setuptools can be pinned to an earlier version for all package builds
+#     that happen in a given job suite, allowing them to succeed. 
+packages:
+  - setuptools 27.2.0
diff --git a/manifests/dev-test.yaml b/manifests/dev-test.yaml
index 531bf80..68da460 100644
--- a/manifests/dev-test.yaml
+++ b/manifests/dev-test.yaml
@@ -7,7 +7,7 @@ channel_URL: 'http://ssb.stsci.edu/astroconda-dev'
 # publication_root path needs to be visible from the slave nodes.
 publication_root: '/eng/ssb/websites/ssbpublic/astroconda-j-dev-testing'
 packages:
-  - fitsverify
+  - drizzlepac
 #  - crds
 #  - cube-tools
 #  - sphinxcontrib-programoutput
diff --git a/manifests/public-test.yaml b/manifests/public-test.yaml
new file mode 100644
index 0000000..331d857
--- /dev/null
+++ b/manifests/public-test.yaml
@@ -0,0 +1,10 @@
+# Recipe repository
+repository: 'https://github.com/astroconda/astroconda-contrib'
+
+# Publication channel to consult when determining what packages already exist.
+channel_URL: 'http://ssb.stsci.edu/astroconda'
+
+# publication_root path needs to be visible from the slave nodes.
+publication_root: '/eng/ssb/websites/ssbpublic/astroconda-j-pub-staging'
+packages:
+  - asdf