Add control of culling behavior via dispatch trigger; more parameterization; formatting

3 files changed, 99 insertions, 67 deletions

diff --git a/jenkins/dispatch.groovy b/jenkins/dispatch.groovy
index 227d1fd..4ba44fd 100644
--- a/jenkins/dispatch.groovy
+++ b/jenkins/dispatch.groovy
@@ -1,27 +1,24 @@
-// The conda version shown in the conda_installers list below is installed
-// Where to obtain this file and the manifest files
-this.build_control_URL = "https://github.com/astroconda/build_control"
-
-// first, then the version is forced to this value.
-this.conda_version = "4.2.15"
-
-// Conda-build is installed fresh at this version.
-this.conda_build_version = "2.1.1"
-
-// Where to get the conda installer
-this.conda_base_URL = "https://repo.continuum.io/miniconda/"
-
-this.recipes_dir = "conda-recipes"
-
-// Support utilities
-this.utils_URL = "https://github.com/rendinam/rambo"
+// Parameters inherited from the calling script via environment injection.
+//----------------------------------------------------------------------------
+// MANIFEST_FILE       - The "release" type; list of recipes/packages to build
+// LABEL               - Node or logical group of build nodes
+// PY_VERSION          - Python version hosted by conda to support the build
+// BUILD_CONTROL_REPO  - Repository holding this & other build system files,
+//                       and manifest files
+// CONDA_VERSION       - First, then the version is forced to this value.
+// CONDA_BUILD_VERSION - Conda-build is installed forced to this version.
+// CONDA_BASE_URL      - Where to get the conda installer
+// UTILS_REPO          - Repository holding support utilities
+
+// Directories to create within the workspace
 this.utils_dir = "utils"
+this.recipes_dir = "conda-recipes"
 
 // The conda installer script to use for various <OS><py_version> combinations.
-this.conda_installers  = ["Linux-py2.7":"Miniconda2-4.2.12-Linux-x86_64.sh",
-                          "Linux-py3.5":"Miniconda3-4.2.12-Linux-x86_64.sh",
-                          "MacOSX-py2.7":"Miniconda2-4.2.12-MacOSX-x86_64.sh",
-                          "MacOSX-py3.5":"Miniconda3-4.2.12-MacOSX-x86_64.sh"]
+this.conda_installers  = ["Linux-py2.7":"Miniconda2-${CONDA_VERSION}-Linux-x86_64.sh",
+                          "Linux-py3.5":"Miniconda3-${CONDA_VERSION}-Linux-x86_64.sh",
+                          "MacOSX-py2.7":"Miniconda2-${CONDA_VERSION}-MacOSX-x86_64.sh",
+                          "MacOSX-py3.5":"Miniconda3-${CONDA_VERSION}-MacOSX-x86_64.sh"]
 
 node(LABEL) {
 
@@ -68,10 +65,17 @@ node(LABEL) {
         assert MANIFEST_FILE != null
         assert MANIFEST_FILE != "manifest_file-DEFAULTVALUE"
 
+        if (CONDA_BASE_URL[-1..-1] == "/") {
+            CONDA_BASE_URL = [0..-2]
+        }
+
+        // Parameters passed at trigger time
+        println("this.cull_manifest: ${this.cull_manifest}")
+
         println("PATH = ${env.PATH}")
 
         // Fetch the manifest files
-        git url: this.build_control_URL
+        git url: BUILD_CONTROL_REPO
 
         // Check for the availability of a download tool and then use it
         // to get the conda installer.
@@ -93,14 +97,14 @@ node(LABEL) {
 
         def conda_installer =
            this.conda_installers["${this.OSname}-py${PY_VERSION}"]
-        dl_cmd = dl_cmd + " ${this.conda_base_URL}${conda_installer}"
+        dl_cmd = dl_cmd + " ${CONDA_BASE_URL}/${conda_installer}"
         sh dl_cmd
 
         // Run miniconda installer and then force to particular version
         sh "bash ./${conda_installer} -b -p miniconda"
         env.PATH = "${env.WORKSPACE}/miniconda/bin/:${env.PATH}"
-        sh "conda install --quiet conda=${this.conda_version}"
-        sh "conda install --quiet --yes conda-build=${this.conda_build_version}"
+        sh "conda install --quiet conda=${CONDA_VERSION}"
+        sh "conda install --quiet --yes conda-build=${CONDA_BUILD_VERSION}"
 
         // Apply bugfix patch to conda_build 2.1.1
         def patches_dir = "${env.WORKSPACE}/patches"
@@ -109,9 +113,8 @@ node(LABEL) {
             sh "patch ${patch}"
         }
 
-        this.manifest = readYaml file: "manifests/" +
-            this.manifest_file
-        if (this.manifest.channel_URL == '/') {
+        this.manifest = readYaml file: "manifests/${MANIFEST_FILE}"
+        if (this.manifest.channel_URL[-1..-1] == "/") {
             this.manifest.channel_URL = this.manifest.channel_URL[0..-2]
         }
         println("Manifest repository: ${this.manifest.repository}")
@@ -132,17 +135,22 @@ node(LABEL) {
     stage("Generate build list") {
         // Obtain build utilities
         dir(this.utils_dir) {
-            git url: this.utils_URL
+            git url: UTILS_REPO
         }
 
-        // Generate a filtered, culled, & dependency-ordered list of available
-        // package recipes.
+        // Generate a filtered, optionally culled, & dependency-ordered list
+        // of available package recipes.
+        def culled_option = "--culled"
+        if (this.cull_manifest == "false") {
+           culled_option = ""
+        }
         def blist_file = "build_list"
         cmd = "${this.utils_dir}/rambo.py"
         args = ["--platform ${this.CONDA_PLATFORM}",
-                "--manifest manifests/${this.manifest_file}",
+                "--python ${PY_VERSION}",
+                "--manifest manifests/${MANIFEST_FILE}",
                 "--file ${blist_file}",
-                "--culled",
+                "${culled_option}",
                 this.recipes_dir]
         for (arg in args) {
             cmd = "${cmd} ${arg}"
@@ -150,13 +158,13 @@ node(LABEL) {
         sh(script: cmd)
 
         def blist_text = readFile blist_file
-        def build_list = blist_text.trim().tokenize()
+        this.build_list = blist_text.trim().tokenize()
         println("Build list:")
-        println(build_list)
+        println(this.build_list)
     }
 
     stage("Build packages") {
-        for (pkg in build_list) {
+        for (pkg in this.build_list) {
             build job: pkg,
               parameters:
                 [string(name: "label", value: env.NODE_NAME),
diff --git a/jenkins/generator_DSL.groovy b/jenkins/generator_DSL.groovy
index 1a77119..a4b40a8 100644
--- a/jenkins/generator_DSL.groovy
+++ b/jenkins/generator_DSL.groovy
@@ -10,7 +10,7 @@ def config = yaml.load(readFileFromWorkspace("manifests/${manifest_file}"))
 
 //-----------------------------------------------------------------------
 // Create a folder to contain the jobs which are created below.
-suite_name = "${manifest_file.tokenize('.')[0]}_${label}_py${py_version}"
+suite_name = "${manifest_file.tokenize(".")[0]}_${label}_py${py_version}"
 folder(suite_name)
 
 
@@ -19,17 +19,35 @@ folder(suite_name)
 // build jobs.
 
 pipelineJob("${suite_name}/_dispatch") {
-    println("label = ${label}")
-    println("manifest_file = ${manifest_file}")
-    println("py_version = ${py_version}")
+    // At trigger-time, allow for setting manifest culling behavior.
+    parameters {
+        booleanParam("cull_manifest",
+                     true,
+                     "Whether or not package recipes that would generate a " +
+                     "package file name that already exists in the manfest's" +
+                     " channel archive are removed from the build list.")
+    }
+    println("MANIFEST_FILE = ${manifest_file}")
+    println("LABEL = ${label}")
+    println("PY_VERSION = ${py_version}")
+    println("BUILD_CONTROL_REPO = ${build_control_repo}")
+    println("CONDA_VERSION = ${conda_version}")
+    println("CONDA_BUILD_VERSION = ${conda_build_version}")
+    println("CONDA_BASE_URL = ${conda_base_URL}")
+    println("UTILS_REPO = ${utils_repo}")
     environmentVariables {
-        env("LABEL", "${label}")
-        env("MANIFEST_FILE", "${manifest_file}")
-        env("PY_VERSION", "${py_version}")
+        env("MANIFEST_FILE", manifest_file)
+        env("LABEL", label)
+        env("PY_VERSION", py_version)
+        env("BUILD_CONTROL_REPO", build_control_repo)
+        env("CONDA_VERSION", conda_version)
+        env("CONDA_BUILD_VERSION", conda_build_version)
+        env("CONDA_BASE_URL", conda_base_URL)
+        env("UTILS_REPO", utils_repo)
     }
     definition {
         cps {
-            script(readFileFromWorkspace('jenkins/dispatch.groovy'))
+            script(readFileFromWorkspace("jenkins/dispatch.groovy"))
             sandbox()
         }
     }
@@ -43,25 +61,25 @@ for(pkg in config.packages) {
 
     pipelineJob("${suite_name}/${pkg}") {
         parameters {
-            stringParam('label',
-                        'label-DEFAULTVALUE',
-                        'The node on which to run.')
-            stringParam('py_version',
-                        'py_version-DEFAULTVALUE',
-                        'python version to use')
-            stringParam('numpy_version',
-                        'numpy_version-DEFAULTVALUE',
-                        'Version of numpy to use')
-            stringParam('parent_workspace',
-                        'parent_workspace-DEFAULTVALUE',
-                        'The workspace dir of the dispatch job')
-            stringParam('manifest_file',
-                        'manifest_file-DEFAULTVALUE',
-                        'Manifest (release) file to use for the build.')
+            stringParam("label",
+                        "label-DEFAULTVALUE",
+                        "The node on which to run.")
+            stringParam("py_version",
+                        "py_version-DEFAULTVALUE",
+                        "python version to use")
+            stringParam("numpy_version",
+                        "numpy_version-DEFAULTVALUE",
+                        "Version of numpy to use")
+            stringParam("parent_workspace",
+                        "parent_workspace-DEFAULTVALUE",
+                        "The workspace dir of the dispatch job")
+            stringParam("manifest_file",
+                        "manifest_file-DEFAULTVALUE",
+                        "Manifest (release) file to use for the build.")
         }
         definition {
             cps {
-                script(readFileFromWorkspace('jenkins/package_builder.groovy'))
+                script(readFileFromWorkspace("jenkins/package_builder.groovy"))
                 sandbox()
             }
         }
diff --git a/jenkins/job-suite-generator.groovy b/jenkins/job-suite-generator.groovy
index 9130d37..9ef3e08 100644
--- a/jenkins/job-suite-generator.groovy
+++ b/jenkins/job-suite-generator.groovy
@@ -16,10 +16,16 @@ node("master") {
         // These variables are provided by the execution of the generator
         // build task with parameters.  Each var is populated by a parameter
         // specification.
-        sh "echo manifest_file=${this.manifest_file}"
-        sh "echo label=${this.label}"
-        sh "echo py_version=${this.py_version}"
-        sh "echo old_jobs_action=${this.old_jobs_action}"
+        println("manifest_file=${this.manifest_file}")
+        println("label=${this.label}")
+        println("py_version=${this.py_version}")
+        println("build_control_repo=${this.build_control_repo}")
+        println("conda_version=${this.conda_version}")
+        println("conda_build_version=${this.conda_build_version}")
+        println("conda_base_URL=${this.conda_base_URL}")
+        println("utils_repo=${this.utils_repo}")
+
+        println("old_jobs_action=${this.old_jobs_action}")
     }
 
     stage("Setup") {
@@ -34,10 +40,10 @@ node("master") {
         sh "cp -r ${env.WORKSPACE}@script/* ."
     }
 
-    stage('Spawn job definitions') {
-        jobDsl targets: ["jenkins/generator_DSL.groovy"].join('\n'),
+    stage("Spawn job definitions") {
+        jobDsl targets: ["jenkins/generator_DSL.groovy"].join("\n"),
                lookupStrategy: "SEED_JOB",
-               additionalClasspath: ["${this.ldir}/*.jar"].join('\n'),
+               additionalClasspath: ["${this.ldir}/*.jar"].join("\n"),
                removeAction: "${this.old_jobs_action}"
     }