Generalize for use within regression test jobs (#14)1.2.2

* Install conda if it's not present during build.

* Purge workspace within node def, and only on first node processed.

* install dir uses workspace base

* Prevent multiple conda installer downloads in a shared environment.

* Update doc

3 files changed, 106 insertions, 8 deletions

diff --git a/README.md b/README.md
index 06a6e2a..29e097a 100644
--- a/README.md
+++ b/README.md
@@ -10,7 +10,6 @@ Functionality provided that extends the native Groovy syntax approach
 
 This library's functionality is automatically made available to every Jenkinsfile hosted in the spacetelescope Github organization.
 
-
 An example job that builds three parallel combinations and runs tests on one of them.
 
 ```groovy
@@ -79,6 +78,7 @@ It has the following members:
 | `conda_packages` | list of strings | no | If this list is defined, the associated build job will create a temporary conda environment to host the job which contains the packages specified. Package specifications are of the form <ul><li>  `<package_name>`  </li><li>  `<package_name>=<version>`  </li></ul>  Example: `bc0.conda_packages = ["pytest", "requests", "numpy=1.14.3"]`  |
 | `conda_override_channels` | boolean | no | Instructs the conda environment creation process to not implicitly prepend the anaconda defaults channel to the list of channels used.  This allows the priority of channels to be used for environment creation to be specified exactly in the order of channels provided in the `conda_channels` list, described below.  If `conda_packages` is not defined in the Jenkinsfile this property is ignored. |
 | `conda_channels` | list of strings | no | The list of channels, in order of search priority, to use when retrieving packages for installation.  If `conda_override_channels` is not defined, this list will have the conda `defaults` channel implicitly prepended to it at installation time.  If `conda_packages` is not defined in the Jenkinsfile this property is ignored.  Example: `bc0.conda_channels = ["http://ssb.stsci.edu/astroconda"]`  |
+ | `conda_ver` | string | no | The version of conda to use when creating environments to host the build. If not supplied, the latest available version of conda will be obtained. NOTE: This may chage from build to build, depending on the state of conda releases. |
  | `env_vars` | list of strings | no | Allow configuration of the shell environment in which build and test commands are run.  Noteworthy behaviors:  <ul><li>  Relative path characters such as `.` and '..' are honored with respect to the isolated build WORKSPACE directory into which the source repository is cloned and the build job takes place.  </li><li>  Shell variables appearing with `$` are dereferenced to their value by the bash shell responsible for hosting the job's activities. The variable name to dereference must exist at the time the environment is created on each parallel node. I.e. variables can appear in the definition of other variables later in the list (the list is processed in order.)  </li><li>  Strings provided in single quotes preclude the need to escape the `$` characters when referencing environment variables. Double quotes require the `$` to be escaped with a `\`.  </li></ul>  |
  | `build_cmds` | list of strings | yes | These commands are run in their order of appearance in this list with the default shell environment and any modifications to that environment provided by the `env_vars` list described above.  <ul><li>  Varables defined in the Jenkinsfile script itself may appear in these commands via `${varname}` notation and are interpolated at script execution time.  </li><li>  These command are executed BEFORE any optional `test_cmds`.  </li></ul>  |
 | `test_cmds` | list of strings | no | These commands are run in their order of appearance in this list with the default shell environment plus any modifications to that environment provided by the `env_vars` list described above.  <ul><li>  If this list is not set for a build configuration, no test commands are run and no test report is generated.  </li><li>  If present, these commands are executed AFTER the build_cmds.  </li></ul> |
diff --git a/src/BuildConfig.groovy b/src/BuildConfig.groovy
index 9eeb3f0..a43ede6 100644
--- a/src/BuildConfig.groovy
+++ b/src/BuildConfig.groovy
@@ -4,9 +4,12 @@ package BuildConfig;
 class BuildConfig implements Serializable {
     def nodetype = ""
     def name = ""
+
     def conda_packages = []
     def conda_override_channels = false
     def conda_channels = []
+    def conda_ver = null
+
     def env_vars = []
     def env_vars_raw = []
     def build_cmds = []
diff --git a/vars/utils.groovy b/vars/utils.groovy
index 54e3ed4..444d8a6 100644
--- a/vars/utils.groovy
+++ b/vars/utils.groovy
@@ -34,6 +34,78 @@ def scm_checkout(args = ['skip_disable':false]) {
     return skip_job
 }
 
+// Returns true if the conda exe is somewhere in the $PATH, false otherwise.
+def conda_present() {
+    def success = sh(script: "conda --version", returnStatus: true)
+    if (success == 0) {
+        return true
+    } else {
+        return false
+    }
+}
+
+
+// Install a particular version of conda by downloading and running the miniconda
+// installer and then installing conda at the specified version.
+//  A version argument of 'null' will result in the latest available conda
+//  version being installed.
+def install_conda(version, install_dir) {
+
+    installer_ver = '4.5.4'
+    default_conda_version = '4.5.4'
+    default_dir = 'miniconda'
+
+    if (version == null) {
+        version = default_conda_version
+    }
+    if (install_dir == null) {
+        install_dir = default_dir
+    }
+
+    def conda_base_url = "https://repo.continuum.io/miniconda"
+
+    def OSname = null
+    def uname = sh(script: "uname", returnStdout: true).trim()
+    if (uname == "Darwin") {
+        OSname = "MacOSX"
+        println("OSname=${OSname}")
+    }
+    if (uname == "Linux") {
+        OSname = uname
+        println("OSname=${OSname}")
+    }
+    assert uname != null
+
+    // Check for the availability of a download tool and then use it
+    // to get the conda installer.
+    def dl_cmds = ["curl -OSs",
+                   "wget --no-verbose --server-response --no-check-certificate"]
+    def dl_cmd = null
+    def stat1 = 999
+    for (cmd in dl_cmds) {
+        stat1 = sh(script: "which ${cmd.tokenize()[0]}", returnStatus: true)
+        if( stat1 == 0 ) {
+            dl_cmd = cmd
+            break
+        }
+    }
+    if (stat1 != 0) {
+        println("Could not find a download tool for obtaining conda. Unable to proceed.")
+        return false
+    }
+
+    def cwd = pwd()
+    def conda_install_dir = "${cwd}/${install_dir}"
+    def conda_installer = "Miniconda3-${installer_ver}-${OSname}-x86_64.sh"
+    dl_cmd = dl_cmd + " ${conda_base_url}/${conda_installer}"
+    if (!fileExists("./${conda_installer}")) {
+        sh dl_cmd
+    }
+
+    // Install miniconda
+    sh "bash ./${conda_installer} -b -p ${install_dir}"
+    return true
+}
 
 // Execute build/test task(s) based on passed-in configuration(s).
 // Each task is defined by a BuildConfig object.
@@ -45,7 +117,7 @@ def scm_checkout(args = ['skip_disable':false]) {
 //                          true when no value is provided.
 def run(configs, concurrent = true) {
     def tasks = [:]
-    for (config in configs) {
+    configs.eachWithIndex { config, index ->
         def BuildConfig myconfig = new BuildConfig() // MUST be inside for loop.
         myconfig = SerializationUtils.clone(config)
         def config_name = ""
@@ -53,17 +125,36 @@ def run(configs, concurrent = true) {
 
         println("config_name: ${config_name}")
 
-        // Code defined within 'tasks' is eventually executed on a separate node.
+        // For containerized CI builds, code defined within 'tasks' is eventually executed
+        // on a separate node.
+        // CAUTION: For builds elsewhere (e.g. nightly regression tests), any parallel
+        //          configs will be executed simultaneously WITHIN THE SAME WORKSPACE.
         // 'tasks' is a java.util.LinkedHashMap, which preserves insertion order.
         tasks["${myconfig.nodetype}/${config_name}"] = {
             node(myconfig.nodetype) {
+                if (index == 0) {
+                    deleteDir()
+                }
                 def runtime = []
                 // If conda packages were specified, create an environment containing
-                // them and then 'activate' it.
+                // them and then 'activate' it. If a specific python version is
+                // desired, it must be specified as a package, i.e. 'python=3.6'
+                // in the list config.conda_packages.
                 if (myconfig.conda_packages.size() > 0) {
-                    def env_name = "tmp_env"
-                    def conda_exe = sh(script: "which conda", returnStdout: true).trim()
+                    // Test for presence of conda. If not available, install it in
+                    // a prefix unique to this build configuration.
+                    def conda_exe = null
+                    if (!conda_present()) {
+                        println('CONDA NOT FOUND. INSTALLING.')
+                        conda_inst_dir = "${env.WORKSPACE}/miniconda-bconf${index}"
+                        install_conda(myconfig.conda_ver, conda_inst_dir)
+                        conda_exe = "${conda_inst_dir}/bin/conda"
+                    } else {
+                        conda_exe = sh(script: "which conda", returnStdout: true).trim()
+                        println('Found conda exe at ${conda_exe}.')
+                    }
                     def conda_root = conda_exe.replace("/bin/conda", "").trim()
+                    def env_name = "tmp_env${index}"
                     def conda_prefix = "${conda_root}/envs/${env_name}".trim()
                     def packages = ""
                     for (pkg in myconfig.conda_packages) {
@@ -71,7 +162,7 @@ def run(configs, concurrent = true) {
                     }
                     // Override removes the implicit 'defaults' channel from the channels
                     // to be used, The conda_channels list is then used verbatim (in
-                    // (priority order) by conda.
+                    // priority order) by conda.
                     def override = ""
                     if (myconfig.conda_override_channels.toString() == 'true') {
                         override = "--override-channels"
@@ -80,7 +171,7 @@ def run(configs, concurrent = true) {
                     for (chan in myconfig.conda_channels) {
                         chans = "${chans} -c ${chan}"
                     }
-                    sh(script: "conda create -q -y -n ${env_name} ${override} ${chans} ${packages}")
+                    sh(script: "${conda_exe} create -q -y -n ${env_name} ${override} ${chans} ${packages}")
                     // Configure job to use this conda environment.
                     myconfig.env_vars.add(0, "CONDA_SHLVL=1")
                     myconfig.env_vars.add(0, "CONDA_PROMPT_MODIFIER=${env_name}")
@@ -130,7 +221,11 @@ def run(configs, concurrent = true) {
                 }
                 withEnv(runtime) {
                     stage("Build (${myconfig.name})") {
+                        println('About to unstash')
+                        sh "pwd"
+                        sh "ls -al"
                         unstash "source_tree"
+                        println('Unstash complete')
                         for (cmd in myconfig.build_cmds) {
                             sh(script: cmd)
                         }