Add initial multi-suite updater.

Simplify operation and downstream parameter passing methodology.

4 files changed, 107 insertions, 46 deletions

diff --git a/jenkins/dispatch.groovy b/jenkins/dispatch.groovy
index 49207c2..89aa26f 100644
--- a/jenkins/dispatch.groovy
+++ b/jenkins/dispatch.groovy
@@ -7,8 +7,7 @@
 // NUMPY_VERSION           - numpy version used to support the build
 // BUILD_CONTROL_REPO      - Repository holding this & other build system files,
 //                           and manifest files
-// BUILD_CONTROL_BRANCH    - Branch to obtain from build control repo
-// BUILD_CONTROL_TAG       - Tag to obtain from build control repo
+// BUILD_CONTROL_GIT_REF   - Git ref to use in obtaining build control scripts
 // CONDA_INSTALLER_VERSION - Conda installer version to use
 // CONDA_VERSION           - conda version is forced to this value
 // CONDA_BUILD_VERSION     - Conda-build is installed forced to this version.
@@ -87,13 +86,11 @@ node(LABEL) {
     sh "env | sort"
 
     // Get the build control files
-    git branch: BUILD_CONTROL_BRANCH, url: BUILD_CONTROL_REPO
+    git url: BUILD_CONTROL_REPO
 
-    // If a tag was specified in the job-suite-generator configuration,
-    // explicitly check out that tag after cloning the (master) branch,
-    // since the 'git' pipeline step does not yet support accessing tags.
-    if (BUILD_CONTROL_TAG != "") {
-        sh(script: "git checkout tags/${BUILD_CONTROL_TAG}")
+    // The 'git' pipeline step does not yet support accessing tags.
+    if (BUILD_CONTROL_GIT_REF != "") {
+        sh(script: "git checkout ${BUILD_CONTROL_GIT_REF}")
     }
 
     // Turn multi-line env var delimiters into simple newlines for
@@ -141,8 +138,7 @@ node(LABEL) {
         "CONDA_BUILD_VERSION: ${CONDA_BUILD_VERSION}\n" +
         "CONDA_BASE_URL: ${CONDA_BASE_URL}\n" +
         "BUILD_CONTROL_REPO: ${BUILD_CONTROL_REPO}\n" +
-        "BUILD_CONTROL_BRANCH: ${BUILD_CONTROL_BRANCH}\n" +
-        "BUILD_CONTROL_TAG: ${BUILD_CONTROL_TAG}\n" +
+        "BUILD_CONTROL_GIT_REF: ${BUILD_CONTROL_GIT_REF}\n" +
         "UTILS_REPO: ${UTILS_REPO}\n" +
         "  Trigger parameters:\n" +
         "this.cull_manifest: ${this.cull_manifest}\n" +
@@ -308,8 +304,7 @@ node(LABEL) {
                 parameters: [
                   string(name: "label", value: env.NODE_NAME),
                   string(name: "build_control_repo", value: BUILD_CONTROL_REPO),
-                  string(name: "build_control_branch", value: BUILD_CONTROL_BRANCH),
-                  string(name: "build_control_tag", value: BUILD_CONTROL_TAG),
+                  string(name: "build_control_git_ref", value: BUILD_CONTROL_GIT_REF),
                   string(name: "py_version", value: PY_VERSION),
                   string(name: "numpy_version", value: NUMPY_VERSION),
                   string(name: "parent_workspace", value: env.WORKSPACE),
diff --git a/jenkins/generator_DSL.groovy b/jenkins/generator_DSL.groovy
index c18085c..0ad86ac 100644
--- a/jenkins/generator_DSL.groovy
+++ b/jenkins/generator_DSL.groovy
@@ -1,5 +1,7 @@
-// Job generator script. Uses Job-DSL plugin API.  // Third party YAML parsing class. Obtain from URL below before use.
-// https://repo1.maven.org/maven2/org/yaml/snakeyaml/1.17/snakeyaml-1.17.jar
+// TODO: Is there a way to reference this local, shaded class instead of fetching it from an external source
+// using @Grab?
+//import org.jenkinsci.plugins.pipeline.utility.steps.shaded.org.yaml.snakeyaml.Yaml // cannot find.
+@Grab('org.yaml:snakeyaml:1.17')
 import org.yaml.snakeyaml.Yaml
 
 def yaml = new Yaml()
@@ -18,23 +20,13 @@ def job_def_generation_time = new Date()
 
 this.script = "dispatch.groovy"
 
-this.build_control_repo = readFileFromWorkspace("VAR-build_control_repo")
-this.build_control_repo = this.build_control_repo.trim()
-
-this.build_control_branch = readFileFromWorkspace("VAR-build_control_branch")
-this.build_control_branch= this.build_control_branch.trim()
-
-this.build_control_tag = readFileFromWorkspace("VAR-build_control_tag")
-this.build_control_tag = this.build_control_tag.trim()
-
 // Keep a specified number of builds (purging those older upon next
 // job execution) for each independent job that is created. This value is set
 // as one of the updater job's parameters.
-//   Note: Since the _dispatch job is
-// executed at the highest frequency, its saved build logs will not go as far
-// back in time for a given value of builds_to_keep than a similar collection
-// of kept logs for a less frequently built job, a slowly-moving package, for
-// instance.
+//   Note: Since the _dispatch job is executed at the highest frequency,
+// its saved build logs will not go as far back in time for a given value of
+// builds_to_keep than a similar collection of kept logs for a less frequently
+// built job, a slowly-moving package, for instance.
 println("builds_to_keep: ${builds_to_keep}")
 this.num_builds_to_keep = builds_to_keep.toInteger()
 
@@ -42,18 +34,17 @@ this.num_builds_to_keep = builds_to_keep.toInteger()
 // over each python version provided by the 'py_versions' job parameter, to obtain
 // every combination of OS and python version. Generate a separate job suite for
 // each combination.
-for (label in labels.trim().tokenize()) {
-    for (py_version in py_versions.trim().tokenize()) {
-        for (numpy_version in numpy_versions.trim().tokenize()) {
-
+for (label in labels) {
+    for (py_version in py_versions) {
+        for (numpy_version in numpy_versions) {
+     
             //-----------------------------------------------------------------------
             // Create a folder to contain the jobs which are created below.
             suite_name = "${manifest_file.tokenize(".")[0]}_${label}_py${py_version}_np${numpy_version}"
             folder(suite_name) {
                 description("Build suite generated: ${job_def_generation_time}\n" +
                             "build control repo: ${build_control_repo}\n" +
-                            "build control branch: ${build_control_branch}\n" +
-                            "build control tag: ${build_control_tag}\n" +
+                            "build control git_ref: ${build_control_git_ref}\n" +
                             "conda installer version: ${conda_installer_version}\n" +
                             "conda version: ${conda_version}\n" +
                             "conda-build version: ${conda_build_version}\n" +
@@ -87,8 +78,7 @@ for (label in labels.trim().tokenize()) {
                 "PY_VERSION: ${py_version}\n" +
                 "NUMPY_VERSION: ${numpy_version}\n" +
                 "BUILD_CONTROL_REPO: ${build_control_repo}\n" +
-                "BUILD_CONTROL_BRANCH: ${build_control_branch}\n" +
-                "BUILD_CONTROL_TAG: ${build_control_tag}\n" +
+                "BUILD_CONTROL_GIT_REF: ${build_control_git_ref}\n" +
                 "CONDA_INSTALLER_VERSION: ${conda_installer_version}\n" +
                 "CONDA_VERSION: ${conda_version}\n" +
                 "CONDA_BUILD_VERSION: ${conda_build_version}\n" +
@@ -103,8 +93,7 @@ for (label in labels.trim().tokenize()) {
                     env("PY_VERSION", py_version)
                     env("NUMPY_VERSION", numpy_version)
                     env("BUILD_CONTROL_REPO", build_control_repo)
-                    env("BUILD_CONTROL_BRANCH", build_control_branch)
-                    env("BUILD_CONTROL_TAG", build_control_tag)
+                    env("BUILD_CONTROL_GIT_REF", build_control_git_ref)
                     env("CONDA_INSTALLER_VERSION", conda_installer_version)
                     env("CONDA_VERSION", conda_version)
                     env("CONDA_BUILD_VERSION", conda_build_version)
@@ -139,12 +128,9 @@ for (label in labels.trim().tokenize()) {
                         stringParam("build_control_repo",
                                     "build_control_repo-DEFAULTVALUE",
                                     "Repository containing the build system scripts.")
-                        stringParam("build_control_branch",
-                                    "build_control_branch-DEFAULTVALUE",
-                                    "Branch checked out to obtain build system scripts.")
-                        stringParam("build_control_tag",
-                                    "build_control_tag-DEFAULTVALUE",
-                                    "Tag checked out to obtain build system scripts.")
+                        stringParam("build_control_git_ref",
+                                    "build_control_git_ref-DEFAULTVALUE",
+                                    "Git ref to use to obtain the build system scripts.")
                         stringParam("py_version",
                                     "py_version-DEFAULTVALUE",
                                     "python version to use")
diff --git a/jenkins/multi_suite_generator.groovy b/jenkins/multi_suite_generator.groovy
new file mode 100644
index 0000000..c2b3ef0
--- /dev/null
+++ b/jenkins/multi_suite_generator.groovy
@@ -0,0 +1,81 @@
+import org.jenkinsci.plugins.pipeline.utility.steps.shaded.org.yaml.snakeyaml.Yaml
+// Top-level pipeline job that provides parameterized machinery for
+// creating one or more build job suites for use in building AstroConda
+// package sets.
+// Uses Job-DSL plugin.
+
+// DSL script path within the repository obtained for this job.
+this.dsl_script = "jenkins/generator_DSL.groovy"
+
+// To sidestep not-serializable exceptions, encapsulate the YAML reading
+// step in a method of its own, used below.
+@NonCPS
+def readYaml(data) {
+  def yaml = new Yaml()
+  payload = yaml.load(data)
+  return payload
+}
+
+def updates = ''
+
+node("master") {
+
+    stage("Prep") {
+
+        // Delete any existing job workspace directory contents.
+        deleteDir()
+        unstash 'build_control'
+        sh "env"
+
+        // Read update/generate YAML description.
+        updates = readYaml(update_list)
+
+        // Clone the manifests repository.
+        dir ("manifests") {
+            git url: manifest_repo
+        }
+
+        build_control_git_ref = git_ref.trim()
+
+        // 'Parameters' variables are provided by the execution of the
+        // generator build task with parameters. Each is populated by a
+        // parameter specification at job execution time. Varaiables defined as
+        // build parameters for this job are automatically available in the
+        // called JobDSL script invoked below by using their base name, i.e.
+        // the name here without a 'this.' prefix. Other variables are not
+        // automatically available, see above.
+        println("  From job config:\n" +
+        "build_control_repo: ${build_control_repo}\n" +
+        "build_control_git_ref: ${build_control_git_ref}\n" +
+        "  Parameters:\n" +
+        "conda_installer_version: ${this.conda_installer_version}\n" +
+        "conda_version: ${this.conda_version}\n" +
+        "conda_build_version: ${this.conda_build_version}\n" +
+        "conda_base_URL: ${this.conda_base_URL}\n" +
+        "utils_repo: ${this.utils_repo}\n" +
+        "old_jobs_action: ${this.old_jobs_action}\n" +
+        "  Other values:\n" +
+        "dsl_script: ${this.dsl_script}")
+    }
+
+    // Loop over requested job suites to generate or update.
+    stage("Spawn job definitions") {
+        def add_params = [:]
+        add_params.put('build_control_repo', build_control_repo)
+        add_params.put('build_control_git_ref', build_control_git_ref)
+
+        updates.keySet().each { key, val ->
+            manifest_file = "${key}.yaml"
+            add_params.put('manifest_file', manifest_file)
+            add_params.put('labels', updates[key].labels)
+            add_params.put('py_versions', updates[key].py_versions)
+            add_params.put('numpy_versions', updates[key].numpy_versions)
+            println("Generating job suite for: ${this.manifest_file}")
+            jobDsl targets: [this.dsl_script].join("\n"),
+                   lookupStrategy: "SEED_JOB",
+                   additionalParameters: add_params,
+                   removeAction: "${this.old_jobs_action}"
+        }
+    }
+
+}
diff --git a/jenkins/package_builder.groovy b/jenkins/package_builder.groovy
index fb7e251..39bb9e0 100644
--- a/jenkins/package_builder.groovy
+++ b/jenkins/package_builder.groovy
@@ -39,8 +39,7 @@ node(this.label) {
         "env.WORKSPACE: ${env.WORKSPACE}\n" +
         "env.JENKINS_HOME: ${env.JENKINS_HOME}\n" +
         "parameter build_control_repo: ${this.build_control_repo}\n" +
-        "parameter build_control_branch: ${this.build_control_branch}\n" +
-        "parameter build_control_tag: ${this.build_control_tag}\n" +
+        "parameter build_control_git_ref: ${this.build_control_git_ref}\n" +
         "parameter parent_workspace: ${this.parent_workspace}\n" +
         "parameter py_version: ${this.py_version}\n" +
         "parameter numpy_version: ${this.numpy_version}\n" +