Merge pull request #11 from rendinam/better_logging

Improve the log output detail and clarity

4 files changed, 132 insertions, 90 deletions

diff --git a/jenkins/dispatch.groovy b/jenkins/dispatch.groovy
index f48022e..4f5551b 100644
--- a/jenkins/dispatch.groovy
+++ b/jenkins/dispatch.groovy
@@ -1,14 +1,15 @@
 // Parameters inherited from the calling script via environment injection.
 //----------------------------------------------------------------------------
-// MANIFEST_FILE       - The "release" type; list of recipes/packages to build
-// LABEL               - Node or logical group of build nodes
-// PY_VERSION          - Python version hosted by conda to support the build
-// BUILD_CONTROL_REPO  - Repository holding this & other build system files,
-//                       and manifest files
-// CONDA_VERSION       - First, then the version is forced to this value.
-// CONDA_BUILD_VERSION - Conda-build is installed forced to this version.
-// CONDA_BASE_URL      - Where to get the conda installer
-// UTILS_REPO          - Repository holding support utilities
+// MANIFEST_FILE        - The "release" type; list of recipes/packages to build
+// LABEL                - Node or logical group of build nodes
+// PY_VERSION           - Python version hosted by conda to support the build
+// BUILD_CONTROL_REPO   - Repository holding this & other build system files,
+//                        and manifest files
+// BUILD_CONTROL_BRANCH - Branch to obtain from build control repo
+// CONDA_VERSION        - First, then the version is forced to this value.
+// CONDA_BUILD_VERSION  - Conda-build is installed forced to this version.
+// CONDA_BASE_URL       - Where to get the conda installer
+// UTILS_REPO           - Repository holding support utilities
 
 // Directories to create within the workspace
 this.utils_dir = "utils"
@@ -26,59 +27,104 @@ node(LABEL) {
     def uname = sh(script: "uname", returnStdout: true).trim()
     if (uname == "Darwin") {
         this.OSname = "MacOSX"
+        this.OSdesc = "MacOS"
+        this.OSversion = sh(script: "sw_vers -productVersion", returnStdout: true).trim()
         env.PATH = "${env.PATH}:/sw/bin"
         this.CONDA_PLATFORM = "osx-64"
     }
     if (uname == "Linux") {
         this.OSname = uname
+        this.OSdesc = sh(script: "lsb_release -i",
+                         returnStdout: true).trim().tokenize()[2]
+        this.OSversion = sh(script: "lsb_release -r",
+                            returnStdout: true).trim().tokenize()[1]
         this.CONDA_PLATFORM = "linux-64"
     }
     assert uname != null
 
-    println("NODE_NAME = ${env.NODE_NAME}")
-
     // Delete any existing job workspace directory contents.
     // The directory deleted is the one named after the jenkins pipeline job.
     deleteDir()
 
+    // Get the manifest and build control files
+    git branch: BUILD_CONTROL_BRANCH, url: BUILD_CONTROL_REPO
+
+    this.manifest = readYaml file: "manifests/${MANIFEST_FILE}"
+    if (this.manifest.channel_URL[-1..-1] == "/") {
+        this.manifest.channel_URL = this.manifest.channel_URL[0..-2]
+    }
+
     // Allow for sharing build_list between stages below.
     this.build_list = []
 
-    stage("Setup") {
+    stage("Summary") {
+        def time = new Date()
+        def manifest_pkg_txt = ""
+        for (pkgname in this.manifest.packages) {
+            manifest_pkg_txt = "${manifest_pkg_txt}${pkgname}\n"
+        }
+        println("\n" +
+        "    Build Info Summary:\n" +
+        "${time}\n" +
+        "JOB_DEF_GENERATION_TIME: ${JOB_DEF_GENERATION_TIME}\n" +
+        "OSname: ${this.OSname}\n" +
+        "OSDesc: ${this.OSdesc}\n" +
+        "OSverion: ${this.OSversion}\n" +
+        "script: dispatch.groovy\n" +
+        "env.WORKSPACE: ${env.WORKSPACE}\n" +
+        "  Job suite parameters:\n" +
+        "LABEL: ${LABEL}\n" +
+        "env.NODE_NAME: ${env.NODE_NAME}\n" +
+        "PY_VERSION: ${PY_VERSION}\n" +
+        "MANIFEST_FILE: ${MANIFEST_FILE}\n" +
+        "PATH: ${PATH}\n" +
+        "CONDA_VERSION: ${CONDA_VERSION}\n" +
+        "CONDA_BUILD_VERSION: ${CONDA_BUILD_VERSION}\n" +
+        "CONDA_BASE_URL: ${CONDA_BASE_URL}\n" +
+        "BUILD_CONTROL_REPO: ${BUILD_CONTROL_REPO}\n" +
+        "BUILD_CONTROL_BRANCH: ${BUILD_CONTROL_BRANCH}\n" +
+        "UTILS_REPO: ${UTILS_REPO}\n" +
+        "  Trigger parameters:\n" +
+        "this.cull_manifest: ${this.cull_manifest}\n" +
+        "  Manifest values:\n" +
+        "Recipe repository: ${this.manifest.repository}\n" +
+        "Numpy version spec: ${this.manifest.numpy_version}\n" +
+        "Channel URL: ${this.manifest.channel_URL}\n" +
+        "Package list:\n${manifest_pkg_txt}")
+    }
 
+    stage("Setup") {
         // Inherited from env() assignment performed in the generator
         // DSL script.
-        println "LABEL = ${LABEL}"
         assert LABEL != null
         assert LABEL != "label-DEFAULTVALUE"
 
         // Inherited from env() assignment performed in the generator
         // DSL script.
-        println("PY_VERSION = ${PY_VERSION}")
         assert PY_VERSION != null
         assert PY_VERSION != "py_version-DEFAULTVALUE"
         this.py_maj_version = "${PY_VERSION.tokenize(".")[0]}"
 
         // Inherited from env() assignment performed in the generator
         // DSL script.
-        println("MANIFEST_FILE = ${MANIFEST_FILE}")
         assert MANIFEST_FILE != null
         assert MANIFEST_FILE != "manifest_file-DEFAULTVALUE"
 
-        if (CONDA_BASE_URL[-1..-1] == "/") {
-            CONDA_BASE_URL = [0..-2]
+        // Get conda recipes
+        dir(this.recipes_dir) {
+            git url: this.manifest.repository
         }
 
-        // Parameters passed at trigger time
-        println("this.cull_manifest: ${this.cull_manifest}")
-
-        println("PATH = ${env.PATH}")
-
-        // Fetch the manifest files
-        git branch: BUILD_CONTROL_BRANCH, url: BUILD_CONTROL_REPO
+        // Get build utilities
+        dir(this.utils_dir) {
+            git url: UTILS_REPO
+        }
 
         // Check for the availability of a download tool and then use it
         // to get the conda installer.
+        if (CONDA_BASE_URL[-1..-1] == "/") {
+            CONDA_BASE_URL = [0..-2]
+        }
         def dl_cmds = ["wget --no-verbose --server-response --no-check-certificate",
                        "curl -OSs"]
         def dl_cmd = null
@@ -100,67 +146,45 @@ node(LABEL) {
         dl_cmd = dl_cmd + " ${CONDA_BASE_URL}/${conda_installer}"
         sh dl_cmd
 
-        // Run miniconda installer and then force to particular version
+        // Install specific versions of miniconda and conda-build
         sh "bash ./${conda_installer} -b -p miniconda"
         env.PATH = "${env.WORKSPACE}/miniconda/bin/:${env.PATH}"
         sh "conda install --quiet conda=${CONDA_VERSION}"
         sh "conda install --quiet --yes conda-build=${CONDA_BUILD_VERSION}"
 
         // Apply bugfix patch to conda_build 2.1.1
-        def filename = "${env.WORKSPACE}/miniconda/lib/python${PY_VERSION}/site-packages/conda_build/config.py"
+        def filename = "${env.WORKSPACE}/miniconda/lib/python${PY_VERSION}/" +
+                       "site-packages/conda_build/config.py"
         def patches_dir = "${env.WORKSPACE}/patches"
         def patchname = "conda_build_2.1.1_substr_fix_py${this.py_maj_version}.patch"
         def full_patchname = "${patches_dir}/${patchname}"
         sh "patch ${filename} ${full_patchname}"
 
-        this.manifest = readYaml file: "manifests/${MANIFEST_FILE}"
-        if (this.manifest.channel_URL[-1..-1] == "/") {
-            this.manifest.channel_URL = this.manifest.channel_URL[0..-2]
-        }
-        println("Manifest repository: ${this.manifest.repository}")
-        println("Manifest numpy version specification: " +
-            "${this.manifest.numpy_version}")
-        println("Manifest channel_URL: ${this.manifest.channel_URL}")
-        println("Manifest packages to build:")
-        for (pkgname in this.manifest.packages) {
-            println(pkgname)
-        }
-
-        // Retrieve conda recipes
-        dir(this.recipes_dir) {
-            git url: this.manifest.repository
-        }
     }
 
     stage("Generate build list") {
-        // Obtain build utilities
-        dir(this.utils_dir) {
-            git url: UTILS_REPO
-        }
-
         // Generate a filtered, optionally culled, & dependency-ordered list
         // of available package recipes.
         def culled_option = "--culled"
         if (this.cull_manifest == "false") {
            culled_option = ""
         }
-        def blist_file = "build_list"
+        def build_list_file = "build_list"
         cmd = "${this.utils_dir}/rambo.py"
         args = ["--platform ${this.CONDA_PLATFORM}",
                 "--python ${PY_VERSION}",
                 "--manifest manifests/${MANIFEST_FILE}",
-                "--file ${blist_file}",
+                "--file ${build_list_file}",
                 "${culled_option}",
                 this.recipes_dir]
         for (arg in args) {
             cmd = "${cmd} ${arg}"
         }
         sh(script: cmd)
-
-        def blist_text = readFile blist_file
-        this.build_list = blist_text.trim().tokenize()
+        def build_list_text = readFile build_list_file
+        this.build_list = build_list_text.trim().tokenize()
         println("Build list:")
-        println(this.build_list)
+        println(build_list_text)
     }
 
     stage("Build packages") {
diff --git a/jenkins/generator_DSL.groovy b/jenkins/generator_DSL.groovy
index bab75a2..de58455 100644
--- a/jenkins/generator_DSL.groovy
+++ b/jenkins/generator_DSL.groovy
@@ -6,10 +6,12 @@ import org.yaml.snakeyaml.Yaml
 
 def yaml = new Yaml()
 def config = yaml.load(readFileFromWorkspace("manifests/${manifest_file}"))
+def job_def_generation_time = new Date() 
 
 
 //-----------------------------------------------------------------------
 // Create a folder to contain the jobs which are created below.
+
 suite_name = "${manifest_file.tokenize(".")[0]}_${label}_py${py_version}"
 folder(suite_name)
 
@@ -18,7 +20,8 @@ folder(suite_name)
 // Generate the dispatch job that will trigger the chain of package
 // build jobs.
 
-pipelineJob("${suite_name}/_dispatch") {
+this.script = "dispatch.groovy"
+pipelineJob("${suite_name}/_${script.tokenize(".")[0]}") {
     // At trigger-time, allow for setting manifest culling behavior.
     parameters {
         booleanParam("cull_manifest",
@@ -27,16 +30,20 @@ pipelineJob("${suite_name}/_dispatch") {
                      "package file name that already exists in the manfest's" +
                      " channel archive are removed from the build list.")
     }
-    println("MANIFEST_FILE = ${manifest_file}")
-    println("LABEL = ${label}")
-    println("PY_VERSION = ${py_version}")
-    println("BUILD_CONTROL_REPO = ${build_control_repo}")
-    println("BUILD_CONTROL_BRANCH = ${build_control_branch}")
-    println("CONDA_VERSION = ${conda_version}")
-    println("CONDA_BUILD_VERSION = ${conda_build_version}")
-    println("CONDA_BASE_URL = ${conda_base_URL}")
-    println("UTILS_REPO = ${utils_repo}")
+    println("\n" +
+    "script: ${this.script}\n" +
+    "MANIFEST_FILE: ${manifest_file}\n" +
+    "LABEL: ${label}\n" +
+    "PY_VERSION: ${py_version}\n" +
+    "BUILD_CONTROL_REPO: ${build_control_repo}\n" +
+    "BUILD_CONTROL_BRANCH: ${build_control_branch}\n" +
+    "CONDA_VERSION: ${conda_version}\n" +
+    "CONDA_BUILD_VERSION: ${conda_build_version}\n" +
+    "CONDA_BASE_URL: ${conda_base_URL}\n" +
+    "UTILS_REPO: ${utils_repo}\n")
     environmentVariables {
+        env("JOB_DEF_GENERATION_TIME", job_def_generation_time)
+        env("SCRIPT", this.script)
         env("MANIFEST_FILE", manifest_file)
         env("LABEL", label)
         env("PY_VERSION", py_version)
@@ -49,7 +56,7 @@ pipelineJob("${suite_name}/_dispatch") {
     }
     definition {
         cps {
-            script(readFileFromWorkspace("jenkins/dispatch.groovy"))
+            script(readFileFromWorkspace("jenkins/${this.script}"))
             sandbox()
         }
     }
@@ -62,6 +69,9 @@ pipelineJob("${suite_name}/_dispatch") {
 for(pkg in config.packages) {
 
     pipelineJob("${suite_name}/${pkg}") {
+        environmentVariables {
+            env("JOB_DEF_GENERATION_TIME", job_def_generation_time)
+        }
         parameters {
             stringParam("label",
                         "label-DEFAULTVALUE",
diff --git a/jenkins/job-suite-generator.groovy b/jenkins/job-suite-generator.groovy
index b5a40c0..cdcb602 100644
--- a/jenkins/job-suite-generator.groovy
+++ b/jenkins/job-suite-generator.groovy
@@ -7,6 +7,10 @@
 // groovy classpath definition prior to imports.
 this.ldir = "libs"
 
+// DSL script path within the repository obtained for this job.
+this.dsl_script = "jenkins/generator_DSL.groovy"
+
+
 node("master") {
 
     stage("Prep") {
@@ -16,17 +20,17 @@ node("master") {
         // These variables are provided by the execution of the generator
         // build task with parameters.  Each var is populated by a parameter
         // specification.
-        println("manifest_file=${this.manifest_file}")
-        println("label=${this.label}")
-        println("py_version=${this.py_version}")
-        println("build_control_repo=${this.build_control_repo}")
-        println("build_control_branch=${this.build_control_branch}")
-        println("conda_version=${this.conda_version}")
-        println("conda_build_version=${this.conda_build_version}")
-        println("conda_base_URL=${this.conda_base_URL}")
-        println("utils_repo=${this.utils_repo}")
-
-        println("old_jobs_action=${this.old_jobs_action}")
+        println("manifest_file: ${this.manifest_file}\n" +
+        "label: ${this.label}\n" +
+        "py_version: ${this.py_version}\n" +
+        "build_control_repo: ${this.build_control_repo}\n" +
+        "build_control_branch: ${this.build_control_branch}\n" +
+        "conda_version: ${this.conda_version}\n" +
+        "conda_build_version: ${this.conda_build_version}\n" +
+        "conda_base_URL: ${this.conda_base_URL}\n" +
+        "utils_repo: ${this.utils_repo}\n" +
+        "old_jobs_action: ${this.old_jobs_action}\n" +
+        "dsl_script: ${this.dsl_script}")
     }
 
     stage("Setup") {
@@ -42,7 +46,7 @@ node("master") {
     }
 
     stage("Spawn job definitions") {
-        jobDsl targets: ["jenkins/generator_DSL.groovy"].join("\n"),
+        jobDsl targets: [this.dsl_script].join("\n"),
                lookupStrategy: "SEED_JOB",
                additionalClasspath: ["${this.ldir}/*.jar"].join("\n"),
                removeAction: "${this.old_jobs_action}"
diff --git a/jenkins/package_builder.groovy b/jenkins/package_builder.groovy
index 3772b66..9f8625e 100644
--- a/jenkins/package_builder.groovy
+++ b/jenkins/package_builder.groovy
@@ -2,24 +2,28 @@ node(this.label) {
 
     dir(this.parent_workspace) {
 
-        println("inherited workspace: ${this.parent_workspace}")
-        println("this.Nodelabel: ${this.label}")
-        println("env.JOB_NAME: ${env.JOB_NAME}")
-        println("env.JOB_BASE_NAME: ${env.JOB_BASE_NAME}")
-        println("env.BUILD_NUMBER: ${env.BUILD_NUMBER}")
-        println("env.NODE_NAME: ${env.NODE_NAME}")
-        println("env.WORKSPACE: ${env.WORKSPACE}")
-        println("env.JENKINS_HOME: ${env.JENKINS_HOME}")
-        println(currentBuild.buildVariables)
-        println("parameter py_version: ${this.py_version}")
-
         env.PATH = "${this.parent_workspace}/miniconda/bin/:" + "${env.PATH}"
-        println("PATH: ${env.PATH}")
+        def time = new Date()
+
+        println("\n" +
+        "    Package Build Info Summary:\n" +
+        "${time}\n" +
+        "JOB_DEF_GENERATION_TIME: ${JOB_DEF_GENERATION_TIME}\n" +
+        "inherited workspace: ${this.parent_workspace}\n" +
+        "this.Nodelabel: ${this.label}\n" +
+        "env.JOB_BASE_NAME: ${env.JOB_BASE_NAME}\n" +
+        "env.JOB_NAME: ${env.JOB_NAME}\n" +
+        "env.BUILD_NUMBER: ${env.BUILD_NUMBER}\n" +
+        "env.NODE_NAME: ${env.NODE_NAME}\n" +
+        "env.WORKSPACE: ${env.WORKSPACE}\n" +
+        "env.JENKINS_HOME: ${env.JENKINS_HOME}\n" +
+        "parameter py_version: ${this.py_version}\n" +
+        "PATH: ${env.PATH}\n")
 
         // Make the log files a bit more deterministic
         env.PYTHONUNBUFFERED = "true"
 
-        // In the directory common to all package build jobs, 
+        // In the directory common to all package build jobs,
         // run conda build --dirty for this package to use any existing work
         // directory or source trees already obtained.
         dir("conda-recipes") {