Generalize for use within regression test jobs (#14)1.2.2

* Install conda if it's not present during build.

* Purge workspace within node def, and only on first node processed.

* install dir uses workspace base

* Prevent multiple conda installer downloads in a shared environment.

* Update doc

1 files changed, 102 insertions, 7 deletions

diff --git a/vars/utils.groovy b/vars/utils.groovy
index 54e3ed4..444d8a6 100644
--- a/vars/utils.groovy
+++ b/vars/utils.groovy
@@ -34,6 +34,78 @@ def scm_checkout(args = ['skip_disable':false]) {
     return skip_job
 }
 
+// Returns true if the conda exe is somewhere in the $PATH, false otherwise.
+def conda_present() {
+    def success = sh(script: "conda --version", returnStatus: true)
+    if (success == 0) {
+        return true
+    } else {
+        return false
+    }
+}
+
+
+// Install a particular version of conda by downloading and running the miniconda
+// installer and then installing conda at the specified version.
+//  A version argument of 'null' will result in the latest available conda
+//  version being installed.
+def install_conda(version, install_dir) {
+
+    installer_ver = '4.5.4'
+    default_conda_version = '4.5.4'
+    default_dir = 'miniconda'
+
+    if (version == null) {
+        version = default_conda_version
+    }
+    if (install_dir == null) {
+        install_dir = default_dir
+    }
+
+    def conda_base_url = "https://repo.continuum.io/miniconda"
+
+    def OSname = null
+    def uname = sh(script: "uname", returnStdout: true).trim()
+    if (uname == "Darwin") {
+        OSname = "MacOSX"
+        println("OSname=${OSname}")
+    }
+    if (uname == "Linux") {
+        OSname = uname
+        println("OSname=${OSname}")
+    }
+    assert uname != null
+
+    // Check for the availability of a download tool and then use it
+    // to get the conda installer.
+    def dl_cmds = ["curl -OSs",
+                   "wget --no-verbose --server-response --no-check-certificate"]
+    def dl_cmd = null
+    def stat1 = 999
+    for (cmd in dl_cmds) {
+        stat1 = sh(script: "which ${cmd.tokenize()[0]}", returnStatus: true)
+        if( stat1 == 0 ) {
+            dl_cmd = cmd
+            break
+        }
+    }
+    if (stat1 != 0) {
+        println("Could not find a download tool for obtaining conda. Unable to proceed.")
+        return false
+    }
+
+    def cwd = pwd()
+    def conda_install_dir = "${cwd}/${install_dir}"
+    def conda_installer = "Miniconda3-${installer_ver}-${OSname}-x86_64.sh"
+    dl_cmd = dl_cmd + " ${conda_base_url}/${conda_installer}"
+    if (!fileExists("./${conda_installer}")) {
+        sh dl_cmd
+    }
+
+    // Install miniconda
+    sh "bash ./${conda_installer} -b -p ${install_dir}"
+    return true
+}
 
 // Execute build/test task(s) based on passed-in configuration(s).
 // Each task is defined by a BuildConfig object.
@@ -45,7 +117,7 @@ def scm_checkout(args = ['skip_disable':false]) {
 //                          true when no value is provided.
 def run(configs, concurrent = true) {
     def tasks = [:]
-    for (config in configs) {
+    configs.eachWithIndex { config, index ->
         def BuildConfig myconfig = new BuildConfig() // MUST be inside for loop.
         myconfig = SerializationUtils.clone(config)
         def config_name = ""
@@ -53,17 +125,36 @@ def run(configs, concurrent = true) {
 
         println("config_name: ${config_name}")
 
-        // Code defined within 'tasks' is eventually executed on a separate node.
+        // For containerized CI builds, code defined within 'tasks' is eventually executed
+        // on a separate node.
+        // CAUTION: For builds elsewhere (e.g. nightly regression tests), any parallel
+        //          configs will be executed simultaneously WITHIN THE SAME WORKSPACE.
         // 'tasks' is a java.util.LinkedHashMap, which preserves insertion order.
         tasks["${myconfig.nodetype}/${config_name}"] = {
             node(myconfig.nodetype) {
+                if (index == 0) {
+                    deleteDir()
+                }
                 def runtime = []
                 // If conda packages were specified, create an environment containing
-                // them and then 'activate' it.
+                // them and then 'activate' it. If a specific python version is
+                // desired, it must be specified as a package, i.e. 'python=3.6'
+                // in the list config.conda_packages.
                 if (myconfig.conda_packages.size() > 0) {
-                    def env_name = "tmp_env"
-                    def conda_exe = sh(script: "which conda", returnStdout: true).trim()
+                    // Test for presence of conda. If not available, install it in
+                    // a prefix unique to this build configuration.
+                    def conda_exe = null
+                    if (!conda_present()) {
+                        println('CONDA NOT FOUND. INSTALLING.')
+                        conda_inst_dir = "${env.WORKSPACE}/miniconda-bconf${index}"
+                        install_conda(myconfig.conda_ver, conda_inst_dir)
+                        conda_exe = "${conda_inst_dir}/bin/conda"
+                    } else {
+                        conda_exe = sh(script: "which conda", returnStdout: true).trim()
+                        println('Found conda exe at ${conda_exe}.')
+                    }
                     def conda_root = conda_exe.replace("/bin/conda", "").trim()
+                    def env_name = "tmp_env${index}"
                     def conda_prefix = "${conda_root}/envs/${env_name}".trim()
                     def packages = ""
                     for (pkg in myconfig.conda_packages) {
@@ -71,7 +162,7 @@ def run(configs, concurrent = true) {
                     }
                     // Override removes the implicit 'defaults' channel from the channels
                     // to be used, The conda_channels list is then used verbatim (in
-                    // (priority order) by conda.
+                    // priority order) by conda.
                     def override = ""
                     if (myconfig.conda_override_channels.toString() == 'true') {
                         override = "--override-channels"
@@ -80,7 +171,7 @@ def run(configs, concurrent = true) {
                     for (chan in myconfig.conda_channels) {
                         chans = "${chans} -c ${chan}"
                     }
-                    sh(script: "conda create -q -y -n ${env_name} ${override} ${chans} ${packages}")
+                    sh(script: "${conda_exe} create -q -y -n ${env_name} ${override} ${chans} ${packages}")
                     // Configure job to use this conda environment.
                     myconfig.env_vars.add(0, "CONDA_SHLVL=1")
                     myconfig.env_vars.add(0, "CONDA_PROMPT_MODIFIER=${env_name}")
@@ -130,7 +221,11 @@ def run(configs, concurrent = true) {
                 }
                 withEnv(runtime) {
                     stage("Build (${myconfig.name})") {
+                        println('About to unstash')
+                        sh "pwd"
+                        sh "ls -al"
                         unstash "source_tree"
+                        println('Unstash complete')
                         for (cmd in myconfig.build_cmds) {
                             sh(script: cmd)
                         }