env 'activation'

Stash all files from master
Selective test run by platform. Start collecting pieces for metadata extraction.
Modify JenkinsfileRT in place.
Load individualized Jenkinsfiles for each pkg
Ensure PATH gets included in env_vars list at appropriate time.
Preprocess env_vars before expanding into runtime list.
Build project if setup.py is present.
Use pip to install projects with a setup.py and ignore their deps.
Generalize PATH manipulation to handle explicit PATH defs and no such defs.
Uninstall conda_packages when done with a single test suite.
Better parsing of conda list info.
Annotate tests with package and build config name.
Remove debugging clause; cleanup

3 files changed, 263 insertions, 27 deletions

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..1377554
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1 @@
+*.swp
diff --git a/JenkinsfileRT b/JenkinsfileRT
new file mode 100644
index 0000000..aac2be4
--- /dev/null
+++ b/JenkinsfileRT
@@ -0,0 +1,58 @@
+// Obtain files from source control system.
+// [skip ci] and [ci skip] have no effect here.
+	if (utils.scm_checkout(['skip_disable':true])) return
+
+// Allow modification of the job configuration, affects all relevant build configs.
+// Pass this object in the argument list to the`run()` function below to apply these settings to the job's execution.
+jobconfig = new JobConfig()
+jobconfig.post_test_summary = true
+
+// Config data to share between builds.
+CFLAGS = ''
+LDFLAGS = ''
+DEFAULT_FLAGS = "${CFLAGS} ${LDFLAGS}"
+// Some waf flags cause a prompt for input during configuration, hence the 'yes'.
+configure_cmd = "yes '' | ./waf configure --prefix=./_install ${DEFAULT_FLAGS}"
+
+// Configure artifactory ingest
+data_config = new DataConfig()
+data_config.server_id = 'bytesalad'
+data_config.root = 'tests_output'
+data_config.match_prefix = '(.*)_result' // .json is appended automatically
+
+// Run nightly tests, which include the slow ones.
+bc = new BuildConfig()
+bc.nodetype = "RHEL-6"
+bc.name = "release"
+bc.env_vars = ['PATH=./_install/bin:$PATH',
+                'OMP_NUM_THREADS=8',
+                'TEST_BIGDATA=https://bytesalad.stsci.edu/artifactory',
+                'jref=/grp/hst/cdbs/jref/',
+                'iref=/grp/hst/cdbs/iref/',
+                'oref=/grp/hst/cdbs/oref/']
+bc.conda_channels = ['http://ssb.stsci.edu/astroconda']
+bc.conda_packages = ['python=3.6',
+                     'cfitsio',
+                     'pkg-config',
+                     'pytest',
+                     'requests',
+                     'astropy',
+                     'ci-watson']
+
+bc.build_cmds = [//"pip install git+https://github.com/spacetelescope/ci_watson.git@master",
+                  "${configure_cmd} --release-with-symbols",
+                  "./waf build",
+                  "./waf install",
+                  "calacs.e --version"]
+//bc.test_cmds = ["pytest tests --basetemp=tests_output --junitxml results.xml --bigdata --slow -v"]
+bc.test_cmds = ["pytest tests/acs/test_hrc_single.py --basetemp=tests_output --junitxml results.xml --bigdata --slow -v"]
+bc.failedUnstableThresh = 1
+bc.failedFailureThresh = 6
+
+// Upload artifacts
+bc.test_configs = [data_config]
+
+// Iterate over configurations that define the (distributed) build matrix.
+// Spawn a host of the given nodetype for each combination and run in parallel.
+// Also apply the job configuration defined in `jobconfig` above.
+   utils.run([bc, jobconfig])
diff --git a/deliver.groovy b/deliver.groovy
index cb45747..5d78e88 100644
--- a/deliver.groovy
+++ b/deliver.groovy
@@ -1,12 +1,30 @@
+import org.jenkinsci.plugins.pipeline.utility.steps.shaded.org.yaml.snakeyaml.Yaml
 @Library('env_utils') _
+library('utils')
+//
+// Note: Jenkinsfiles for projects that produce dev packages to be used
+//       in the assembling of delivery sets must have the build config
+//       used to run the intended regression test suite named 'bc'.
+//
 // Parameters made available from Jenkins job configuration:
 //   delivery_name
+//   python_version
 //   aux_packages
 //   conda_installer_version
 //   conda_version
 //   output_dir
 
-def gen_specfiles(label) {
+
+// To sidestep not-serializable exceptions, encapsulate the YAML reading
+// step in a method of its own, used below.
+@NonCPS
+def readYaml(data) {
+  def yaml = new Yaml()
+  payload = yaml.load(data)
+  return payload
+}
+
+def gen_specfiles(label, run_tests) {
     
     node(label) {
 
@@ -14,6 +32,9 @@ def gen_specfiles(label) {
         // The directory deleted is the one named after the jenkins pipeline job.
         deleteDir()
 
+        println("PATH ----")
+        println("${env.PATH}")
+
         unstash "script"
 
         env.PYTHONPATH = ""
@@ -23,29 +44,195 @@ def gen_specfiles(label) {
         def WORKDIR = pwd()
         println("WORKDIR = ${WORKDIR}")
 
-        conda.install()
-
-        PATH = "${WORKDIR}/miniconda/bin:${PATH}"
-        def cpkgs = "conda=${conda_version}"
-        def pkg_list = aux_packages.replaceAll('\n', ' ')
-
         def flags = ""
         println("${finalize} - finalize")
         if ("${finalize}" == "true") {
             flags = "--final"
         }
 
-        withEnv(["HOME=${HOME}", "PATH=${PATH}"]) {
+        conda.install()
+
+        path_with_conda = "${WORKDIR}/miniconda/bin:${PATH}"
+        def cpkgs = "conda=${conda_version}"
+        def pkg_list = aux_packages.replaceAll('\n', ' ')
+       
+
+        withEnv(["HOME=${WORKDIR}", "PATH=${path_with_conda}"]) {
+            sh "env | sort"
+            conda_exe = sh(script: "which conda", returnStdout: true).trim()
+            println("Found conda exe at ${conda_exe}.")
+            conda_root = conda_exe.replace("/bin/conda", "").trim()
             sh "conda install --quiet --yes ${cpkgs}"
 
             // Generate spec files
-            sh "${WORKDIR}/mktestenv.sh -n ${delivery_name} -p 3.5 ${flags} ${pkg_list}"
-            sh "${WORKDIR}/mktestenv.sh -n ${delivery_name} -p 3.6 ${flags} ${pkg_list}"
+            sh "${WORKDIR}/mktestenv.sh -n ${delivery_name} -p ${python_version} ${flags} ${pkg_list}"
        
             // Make spec files available to master node. 
             stash name: "spec-stash-${label}", includes: "hstdp*.txt"
+
+            // List environments for testing
+            sh "conda env list"
         }
-    }
+
+        // Get source repo for each aux package _at the commit used to
+        // build the pkg_.
+        // For each package in pkg_list, locate installed package in local conda
+        // dir, extract metadata to obtain:
+        //    commit hash
+        //    source repository
+        if (run_tests) {
+            def env_name = sh(script: "ls hstdp*", returnStdout: true).trim()[0..-5]
+            println("env_name: ${env_name}")
+            for (pkg in pkg_list.tokenize()) {
+                println("Extracting metadata for ${pkg}...")
+                pkg_name = pkg.tokenize('=')[0]
+                println("pkg_name: ${pkg_name}")
+                pkg_version = pkg.tokenize('=')[1]
+                ccmd = "${conda_exe} list -n ${env_name} | grep ${pkg_name} | grep dev"
+                pkg_info = sh(script: ccmd,
+                               returnStdout: true).trim()
+                println("pkg_info: ${pkg_info}")
+                meta_base = pkg_info.tokenize()[1]
+                meta_rev = pkg_info.tokenize()[2]
+                meta_dir = "${pkg_name}-${meta_base}-${meta_rev}"
+                metafile = "${conda_root}/pkgs/${meta_dir}/info/recipe/meta.yaml"
+                filedata = readFile(metafile)
+                meta = readYaml(filedata)
+                git_url = meta['source'].git_url
+
+                // Use this clone to run the full test suite.
+                sh "git clone ${git_url} ./${pkg_name}"
+
+                // For development only. Disable before use.
+                //if (pkg_name == "hstcal") {
+                //    jenkinsfile = "./JenkinsfileRT"
+                //} else {
+                    jenkinsfile = "${WORKDIR}/${pkg_name}/JenkinsfileRT"
+                //}
+
+                // Post-process each -dev project's JenkinsfileRT to allow
+                // importing of only the configuration values without running
+                // the processing machinery called within which would complicate
+                // matters here. 
+                // Disable scm_checkout call in project's JenkinsfileRT.
+                // NOTE: 'if' statement must be on same line as scm_checkout call.
+                // TODO: Generalize to make this more robust against layout
+                //       changes in file.
+                sh "sed -i 's/^\\s*if/\\/\\/if/' ${jenkinsfile}" 
+                // Disable utils.run line in projects's JenkinsfileRT.
+                sh "sed -i 's/^\\s*utils.run/\\/\\/utils.run/' ${jenkinsfile}"
+
+                println("About lo load local modified JenkinsfileRT")
+                sh "ls -al"
+                // Add declarations from file to this namespace. Provides 'bc'.
+                // TODO: Iterate through namespace to locate buildConfig
+                // objects without a standard name?
+                // NOTE: 'evaluate' runs the methods it encounters.
+                //       Edit to disable to prevent undesired side effects.
+                jf = evaluate readFile(jenkinsfile)
+
+                def conda_prefix = "${conda_root}/envs/${env_name}".trim()
+
+                def path_found = false
+                for (var in bc.env_vars) {
+                    println("--> ${var}")
+                    if (var[0..3] == 'PATH') {
+                        path_found = true
+                        pathvar_idx = bc.env_vars.indexOf(var)
+                        bc.env_vars.remove(pathvar_idx)
+                        println(" --- ADJUSTING PATH FOR CONDA ENV ----")
+                        var = "PATH=${conda_prefix}/bin:${WORKDIR}/miniconda/bin:${var[5..-1]}"
+                        println(var)
+                        bc.env_vars.add(var)
+                    }
+                }
+                if (!path_found) {
+                    println("--- Adding PATH prefix for conda environment and conda exe.")
+                    bc.env_vars.add("PATH=${conda_prefix}/bin:${WORKDIR}/miniconda/bin:\$PATH")
+                }
+                bc.env_vars.add("ENVIRONMENT=VARIABLE-${pkg}")
+
+                def bconfig = utils.expandEnvVars(bc) // expand vars into .runtime
+
+                def env_vars = bconfig.runtime
+                // 'Activate' conda env and operate within it.        
+                env_vars.add(0, "CONDA_SHLVL=1")
+                env_vars.add(0, "CONDA_PROMPT_MODIFIER=${env_name}")
+                env_vars.add(0, "CONDA_EXE=${conda_exe}")
+                env_vars.add(0, "CONDA_PREFIX=${conda_prefix}")
+                env_vars.add(0, "CONDA_PYTHON_EXE=${conda_prefix}/bin/python")
+                env_vars.add(0, "CONDA_DEFAULT_ENV=${env_name}")
+
+                println('FINAL ENV VARS')
+                for (var in env_vars) {
+                    println(var)
+                }
+
+                // Run testing commands
+                withEnv(env_vars) {
+                    println("============= withEnv ==========")
+                    sh "env | sort"
+                    // Obtain pip for build/install use
+                    sh "conda install pip"
+                    dir(pkg_name) {
+                       sh "ls -l"
+                       def conda_pkgs = ""
+                       for (cpkg in bc.conda_packages) {
+                           conda_pkgs = "${conda_pkgs} ${cpkg} "
+                       }
+                       sh "conda install ${conda_pkgs} pytest -c http://ssb.stsci.edu/astroconda -c defaults -q -y"
+
+                       // If setup.py exists in project, run `python setup.py build` to prepare
+                       // local source tree for testing. This is required for projects with C
+                       // extensions.
+                       if (fileExists("setup.py")) {
+                           sh(script: "pip install --no-deps -e .")
+                       }
+                       println("Test commands(s):")
+                       println(bc.test_cmds)
+                       for (tcmd in bc.test_cmds) {
+                           sh(script: "${tcmd} || true")
+                       }
+                       // Uninstall packages pulled in for this test run.
+                       sh "conda remove ${conda_pkgs} --force -y"
+
+                       // Read in test reports for display.
+                       // TODO: Use shared lib for this functionality.
+                       // Process the results file to include the build config name as a prefix
+                       // on each test name so that it is obvious from where each test result
+                       // comes.
+                       report_exists = sh(script: "test -e *.xml", returnStatus: true)
+                       if (report_exists == 0) {
+                           repfile = sh(script:"find *.xml", returnStdout: true).trim()
+                           command = "cp ${repfile} ${repfile}.modified"
+                           sh(script:command)
+                           sh(script:"sed -i 's/ name=\"/ name=\"[${pkg_name}:${bc.name}] /g' *.xml.modified")
+                           step([$class: 'XUnitBuilder',
+                               thresholds: [
+                               [$class: 'SkippedThreshold', unstableThreshold: "${bc.skippedUnstableThresh}"],
+                               [$class: 'SkippedThreshold', failureThreshold: "${bc.skippedFailureThresh}"],
+                               [$class: 'FailedThreshold', unstableThreshold: "${bc.failedUnstableThresh}"],
+                               [$class: 'FailedThreshold', failureThreshold: "${bc.failedFailureThresh}"]],
+                               tools: [[$class: 'JUnitType', pattern: '*.xml.modified']]])
+                       } else {
+                           println("No .xml files found in directory ${pkg_name}. Test report ingestion skipped.")
+                       }
+                    }
+                }
+            } // endfor pkg
+        }
+
+        stage('archive') {
+            hostname = sh(script: "hostname", returnStdout: true).tokenize(".")[0]
+            println(hostname)
+            withCredentials([usernamePassword(credentialsId: '322ad15d-2f5b-4d06-87fa-b45a88596f30',
+                usernameVariable: 'USERNAME',
+                passwordVariable: 'PASSWORD')]) {
+                    sh "rsync -avzr hstdp*.txt ${USERNAME}@${hostname}:${output_dir}"
+               }
+        }
+
+    } // end node()
 }
 
 
@@ -59,23 +246,13 @@ node('master') {
                        numToKeepStr: '4')), pipelineTriggers([])])
     stage('create specfiles') {
         deleteDir()
-        sh "cp -r ${WORKSPACE}@script/*.sh ."
-        stash name: "script", includes: "*.sh"
+        sh "cp -r ${WORKSPACE}@script/* ."
+        // Carry delivery_control files into each remote node's workspace.
+        stash name: "script", includes: "*"
         parallel(
-            Linux: { gen_specfiles('RHEL-6') },
-            MacOS: { gen_specfiles('OSX-10.11') }
+            // Generate spec files. Only run tests on Linux (for now).
+            Linux: { gen_specfiles('RHEL-6', true) },
+            //MacOS: { gen_specfiles('OSX-10.13', false) }
         )
     }
-
-    stage('archive') {
-        // Retrieve the spec files from the nodes where they were created.
-        unstash "spec-stash-RHEL-6"
-        unstash "spec-stash-OSX-10.11"
-        hostname = sh(script: "hostname", returnStdout: true).tokenize(".")[0]
-        withCredentials([usernamePassword(credentialsId: '322ad15d-2f5b-4d06-87fa-b45a88596f30',
-            usernameVariable: 'USERNAME',
-            passwordVariable: 'PASSWORD')]) {
-                sh "rsync -avzr hstdp*.txt ${USERNAME}@${hostname}:${output_dir}"
-           }
-    }
 }