* Only publish packages that have successful build AND test stages. Previously, if a package file was created, it was published even if the tests did not all pass.

* Refactor build status propagation
   * Allow for a trigger-level summary of all downstream job results.
   * Simplify machinery by using result paylods provided directly by build calls.

2 files changed, 70 insertions, 49 deletions

diff --git a/jenkins/dispatch.groovy b/jenkins/dispatch.groovy
index f3981a1..473587f 100644
--- a/jenkins/dispatch.groovy
+++ b/jenkins/dispatch.groovy
@@ -19,8 +19,6 @@
 this.utils_dir = "utils"
 this.recipes_dir = "conda-recipes"
 
-this.build_status_file = "propagated_build_status"
-
 // The conda installer script to use for various <OS><py_version> combinations.
 this.conda_installers  = ["Linux-py2":"Miniconda2-${CONDA_INSTALLER_VERSION}-Linux-x86_64.sh",
                           "Linux-py3":"Miniconda3-${CONDA_INSTALLER_VERSION}-Linux-x86_64.sh",
@@ -294,36 +292,54 @@ node(LABEL) {
         this.build_list = build_list_text.trim().tokenize()
         println("Build list:")
         println(build_list_text)
-
-        // Write build status file to facilitate build status propagation
-        // from child jobs.
-        sh "echo SUCCESS > ${this.build_status_file}"
     }
 
     stage("Build packages") {
+        overall_result = "SUCCESS"
+        build_objs = [:]
+        def build_obj = null
         for (pkg in this.build_list) {
-            build job: pkg,
-              parameters: [
-                 string(name: "label", value: env.NODE_NAME),
-                 string(name: "build_control_repo", value: BUILD_CONTROL_REPO),
-                 string(name: "build_control_branch", value: BUILD_CONTROL_BRANCH),
-                 string(name: "build_control_tag", value: BUILD_CONTROL_TAG),
-                 string(name: "py_version", value: PY_VERSION),
-                 string(name: "numpy_version", value: NUMPY_VERSION),
-                 string(name: "parent_workspace", value: env.WORKSPACE),
-                 string(name: "manifest_file", value: MANIFEST_FILE),
-                 string(name: "cull_manifest", value: this.cull_manifest),
-                 string(name: "channel_URL", value: this.manifest.channel_URL),
-                 string(name: "use_version_pins", value: this.use_version_pins),
-                 text(name: "supp_env_vars", value: this.supp_env_vars)
-              ],
-              propagate: false
+            build_objs[pkg] = build(
+                job: pkg,
+                parameters: [
+                  string(name: "label", value: env.NODE_NAME),
+                  string(name: "build_control_repo", value: BUILD_CONTROL_REPO),
+                  string(name: "build_control_branch", value: BUILD_CONTROL_BRANCH),
+                  string(name: "build_control_tag", value: BUILD_CONTROL_TAG),
+                  string(name: "py_version", value: PY_VERSION),
+                  string(name: "numpy_version", value: NUMPY_VERSION),
+                  string(name: "parent_workspace", value: env.WORKSPACE),
+                  string(name: "manifest_file", value: MANIFEST_FILE),
+                  string(name: "cull_manifest", value: this.cull_manifest),
+                  string(name: "channel_URL", value: this.manifest.channel_URL),
+                  string(name: "use_version_pins", value: this.use_version_pins),
+                  text(name: "supp_env_vars", value: this.supp_env_vars)
+                ],
+                propagate: false)
+            // Ratchet up the overall build result status if necessary.
+            // Set overall status to the worst propagated from individual package jobs.
+            // 'FAILURE' is worse than 
+            // 'UNSTABLE' which is worse than 
+            // 'SUCCESS'.
+            def result = build_objs[pkg].result
+            if (result != "SUCCESS") {
+                if (result == "FAILURE") {
+                    overall_result = "FAILURE"
+                }
+                if (result == "UNSTABLE" && overall_result == "SUCCESS") {
+                    overall_result = "UNSTABLE"
+                }
+            }
         }
-        // Set overall status to that propagated from individual jobs.
-        // This will be the most severe status encountered in all sub jobs.
-        def tmp_status = readFile this.build_status_file
-        tmp_status = tmp_status.trim()
-        currentBuild.result = tmp_status
+        currentBuild.result = overall_result
+
+        // Print summary of results from each package build job.
+        def results_msg = ""
+        build_objs.each{
+            key, value -> results_msg = "${results_msg}${key} : ${value.result}\n"
+        }
+        println(results_msg)
+        currentBuild.description = results_msg
     }
 
     stage ("Publish") {
@@ -332,20 +348,19 @@ node(LABEL) {
         def artifacts_present =
             sh(script: "ls ${this.conda_build_output_dir}/*.tar.bz2 >/dev/null 2>&1",
                returnStatus: true)
-        def rsync_cmd = "rsync -avzr --ignore-existing"
+        def rsync_cmd = "rsync -avzr"
         if (artifacts_present == 0) {
             sh(script: "${rsync_cmd} ${this.conda_build_output_dir}/*.tar.bz2 ${publication_path}")
             // Use a lock file to prevent two dispatch jobs that finish at the same
             // time from trampling each other's indexing process.
-            def lockfile = "${publication_path}/LOCK-Jenkins"
-            def file = new File(lockfile)
             def tries_remaining = this.max_publication_tries
-            if (file.exists()) {
+            def lockfile = "${publication_path}/LOCK-Jenkins"
+            if ( fileExists(lockfile) ) {
                 println("Lockfile already exists, waiting for it to be released...")
                 while ( tries_remaining > 0) {
                     println("Waiting ${this.publication_lock_wait_s}s for lockfile release...")
                     sleep(this.publication_lock_wait_s)
-                    if ( !file.exists() ) {
+                    if ( !fileExists(file) ) {
                         break
                     }
                     tries_remaining--
@@ -397,10 +412,8 @@ node(LABEL) {
             sh(script: cmd)
 
             short_plat = CONDA_PLATFORM.tokenize("-")[0]
-            //short_py_ver = PY_VERSION.replaceAll(".", "")
             short_py_ver = "${PY_VERSION[0]}${PY_VERSION[2]}"
             specfile_name = "${specfile_type}-${jwst_git_rev}-${short_plat}-py${short_py_ver}.00.txt"
-            //outdir = "/eng/ssb/websites/ssbpublic/astroconda-releases-staging"
             outdir = specfile_output
             outfile = "${outdir}/${specfile_name}"
             sh(script: "conda list -n spec --explicit > ${outfile}")
diff --git a/jenkins/package_builder.groovy b/jenkins/package_builder.groovy
index 51ff9c8..fb7e251 100644
--- a/jenkins/package_builder.groovy
+++ b/jenkins/package_builder.groovy
@@ -2,8 +2,6 @@
 //----------------------------------------------------------------------------
 // CONDA_BUILD_VERSION  - Conda-build is installed forced to this version.
 
-this.build_status_file = "${this.parent_workspace}/propagated_build_status"
-
 node(this.label) {
 
     // Add any supplemental environment vars to the build environment.
@@ -19,7 +17,7 @@ node(this.label) {
 
         env.PATH = "${this.parent_workspace}/miniconda/bin/:" + "${env.PATH}"
         env.PYTHONPATH = ""
-        // Make the log files a bit more deterministic
+        // Make the output a bit more deterministic
         env.PYTHONUNBUFFERED = "true"
         def time = new Date()
 
@@ -54,9 +52,6 @@ node(this.label) {
         "PYTHONPATH: ${env.PYTHONPATH}\n" +
         "PYTHONUNBUFFERED: ${env.PYTHONUNBUFFERED}\n")
 
-        def build_status = readFile this.build_status_file
-        build_status = build_status.trim()
-
         // In the directory common to all package build jobs,
         // run conda build --dirty for this package to use any existing work
         // directory or source trees already obtained.
@@ -103,11 +98,6 @@ node(this.label) {
                           returnStatus: true)
                 if (stat != 0) {
                     currentBuild.result = "FAILURE"
-                    // Ratchet up the overall build status severity if this
-                    // is the most severe seen so far.
-                    if (build_status != "FAILURE") {
-                            sh "echo ${currentBuild.result} > ${this.build_status_file}"
-                    }
                 }
             }
 
@@ -119,7 +109,7 @@ node(this.label) {
                             "--python=${this.py_version}",
                             "--numpy=${this.numpy_version}",
                             "--override-channels"]
-                    // Channel arguments are order-dependent.
+                    // NOTE: Channel arguments are order-sensitive.
                     if (this.cull_manifest) {
                         args.add("--channel ${this.channel_URL}")
                     }
@@ -134,12 +124,30 @@ node(this.label) {
                     stat = 999
                     stat = sh(script: "${build_cmd} ${env.JOB_BASE_NAME}",
                               returnStatus: true)
+
                     if (stat != 0) {
                         currentBuild.result = "UNSTABLE"
                         // Ratchet up the overall build status severity if this
-                        // is the most severe seen so far.
-                        if (build_status == "SUCCESS") {
-                            sh "echo ${currentBuild.result} > ${this.build_status_file}"
+                        // is the most severe status seen so far.
+                        // Also, delete the package file so that it cannot be
+                        // published. The package file to remove is the most
+                        // recent .tar.bz2 file in the build output directory.
+                        bld_dir = "${this.parent_workspace}/miniconda/conda-bld"
+
+                        // Get the most recently created package name.
+                        def plat_dir = "${bld_dir}/linux-64"
+                        if (!fileExists(plat_dir)) {
+                            plat_dir = "${bld_dir}/osx-64"
+                        }
+                        cmd = "ls -t ${plat_dir}/*.tar.bz2 | head -n1"
+                        def pkg_full_name = sh(script: cmd, returnStdout: true)
+
+                        println("Deleting file ${pkg_full_name}")
+                        // Use shell call here because file.exists() and file.delete()
+                        // simply don't work correctly and report no errors to that effect.
+                        stat = sh(script: "rm -f ${pkg_full_name}", returnStatus: true)
+                        if (stat != 0) {
+                            println("ERROR deleting package file ${pkg_full_name}")
                         }
                     }
                 }