Merge pull request #22 from rendinam/artifacts

Publish packages

5 files changed, 77 insertions, 9 deletions

diff --git a/jenkins/dispatch.groovy b/jenkins/dispatch.groovy
index 7dd09e0..551bad4 100644
--- a/jenkins/dispatch.groovy
+++ b/jenkins/dispatch.groovy
@@ -23,6 +23,10 @@ this.conda_installers  = ["Linux-py2.7":"Miniconda2-${CONDA_VERSION}-Linux-x86_6
                           "MacOSX-py2.7":"Miniconda2-${CONDA_VERSION}-MacOSX-x86_64.sh",
                           "MacOSX-py3.5":"Miniconda3-${CONDA_VERSION}-MacOSX-x86_64.sh"]
 
+// Values controlling the conda index stage which happens after any packages are created.
+this.max_publication_tries = 5
+this.publication_lock_wait_s = 10
+
 node(LABEL) {
 
     this.OSname = null
@@ -44,11 +48,15 @@ node(LABEL) {
     }
     assert uname != null
 
+    // Conda paths
+    this.conda_install_dir = "${env.WORKSPACE}/miniconda"
+    this.conda_build_output_dir = "${this.conda_install_dir}/conda-bld/${this.CONDA_PLATFORM}"
+
     env.PYTHONPATH = ""
     // Make the log files a bit more deterministic
     env.PYTHONUNBUFFERED = "true"
 
-    sh "printenv"
+    sh "env | sort"
 
     // Delete any existing job workspace directory contents.
     // The directory deleted is the one named after the jenkins pipeline job.
@@ -157,8 +165,8 @@ node(LABEL) {
         sh dl_cmd
 
         // Install specific versions of miniconda and conda-build
-        sh "bash ./${conda_installer} -b -p miniconda"
-        env.PATH = "${env.WORKSPACE}/miniconda/bin:${env.PATH}"
+        sh "bash ./${conda_installer} -b -p ${this.conda_install_dir}"
+        env.PATH = "${this.conda_install_dir}/bin:${env.PATH}"
         def cpkgs = "conda=${CONDA_VERSION} conda-build=${CONDA_BUILD_VERSION}"
         sh "conda install --quiet --yes ${cpkgs} python=${PY_VERSION}"
 
@@ -167,7 +175,7 @@ node(LABEL) {
         def conda_build_maj_ver = conda_build_version.tokenize()[1].tokenize('.')[0]
         if (conda_build_maj_ver == "2") {
             println("conda-build major version ${conda_build_maj_ver} detected. Applying bugfix patch.")
-            def filename = "${env.WORKSPACE}/miniconda/lib/python${PY_VERSION}/" +
+            def filename = "${this.conda_install_dir}/lib/python${PY_VERSION}/" +
                            "site-packages/conda_build/config.py"
             def patches_dir = "${env.WORKSPACE}/patches"
             def patchname = "conda_build_2.1.1_substr_fix_py${this.py_maj_version}.patch"
@@ -231,5 +239,40 @@ node(LABEL) {
         tmp_status = tmp_status.trim()
         currentBuild.result = tmp_status
     }
+
+    stage ("Publish") {
+        def publication_path = "${PUBLICATION_ROOT}/${this.CONDA_PLATFORM}"
+        // Copy and index packages if any were produced in the build.
+        def artifacts_present =
+            sh(script: "ls ${this.conda_build_output_dir}/*.tar.bz2 >/dev/null 2>&1",
+               returnStatus: true)
+        println("artifacts present = ${artifacts_present}")
+        if (artifacts_present == 0) {
+            sh(script: "rsync -avzr ${this.conda_build_output_dir}/*.tar.bz2 ${publication_path}")
+            // Use a lock file to prevent two dispatch jobs that finish at the same
+            // time from trampling each other's indexing process.
+            def lockfile = "${publication_path}/LOCK-Jenkins"
+            def file = new File(lockfile)
+            def tries_remaining = this.max_publication_tries
+            if (file.exists()) {
+                println("Lockfile already exists, waiting for it to be released...")
+                while ( tries_remaining > 0) {
+                    println("Waiting ${this.publication_lock_wait_s}s for lockfile release...")
+                    sleep(this.publication_lock_wait_s * 1000)
+                    if ( !file.exists() ) {
+                        break
+                    }
+                    tries_remaining--
+                }
+            }
+            if (tries_remaining != 0) {
+                sh(script: "touch ${lockfile}")
+                dir(this.conda_build_output_dir) {
+                    sh(script: "conda index ${publication_path}")
+                }
+                sh(script: "rm -f ${lockfile}")
+            }
+        }
+    }
 }
 
diff --git a/jenkins/generator_DSL.groovy b/jenkins/generator_DSL.groovy
index f74275c..f05c46c 100644
--- a/jenkins/generator_DSL.groovy
+++ b/jenkins/generator_DSL.groovy
@@ -46,7 +46,8 @@ pipelineJob("${suite_name}/_${script.tokenize(".")[0]}") {
     "CONDA_VERSION: ${conda_version}\n" +
     "CONDA_BUILD_VERSION: ${conda_build_version}\n" +
     "CONDA_BASE_URL: ${conda_base_URL}\n" +
-    "UTILS_REPO: ${utils_repo}\n")
+    "UTILS_REPO: ${utils_repo}\n" +
+    "PUBLICATION_ROOT: ${publication_root}\n")
     environmentVariables {
         env("JOB_DEF_GENERATION_TIME", job_def_generation_time)
         env("SCRIPT", this.script)
@@ -59,6 +60,7 @@ pipelineJob("${suite_name}/_${script.tokenize(".")[0]}") {
         env("CONDA_BUILD_VERSION", conda_build_version)
         env("CONDA_BASE_URL", conda_base_URL)
         env("UTILS_REPO", utils_repo)
+        env("PUBLICATION_ROOT", publication_root)
     }
     definition {
         cps {
diff --git a/jenkins/package_builder.groovy b/jenkins/package_builder.groovy
index 8ddbe9f..436b4b2 100644
--- a/jenkins/package_builder.groovy
+++ b/jenkins/package_builder.groovy
@@ -10,7 +10,7 @@ node(this.label) {
         env.PYTHONUNBUFFERED = "true"
         def time = new Date()
 
-        sh "printenv"
+        sh "env | sort"
 
         println("\n" +
         "    Package Build Info Summary:\n" +
diff --git a/manifests/dev-test.yaml b/manifests/dev-test.yaml
new file mode 100644
index 0000000..6bdf084
--- /dev/null
+++ b/manifests/dev-test.yaml
@@ -0,0 +1,15 @@
+repository: 'https://github.com/astroconda/astroconda-dev'
+channel_URL: 'http://ssb.stsci.edu/astroconda-dev'
+numpy_version: 1.13
+packages:
+  - crds
+  - cube-tools
+  - sphinxcontrib-programoutput
+  - pyds9
+  - wcstools
+  - webbpsf
+#  - mosviz
+#  - specutils
+#  - glueviz
+#  - glue-vispy-viewers
+#  - glue-core
diff --git a/manifests/test.yaml b/manifests/test.yaml
index 0c626c2..4d6806e 100644
--- a/manifests/test.yaml
+++ b/manifests/test.yaml
@@ -1,9 +1,17 @@
-repository: 'https://github.com/astroconda/astroconda-contrib'
-channel_URL: 'http://ssb.stsci.edu/astroconda'
+repository: 'https://github.com/astroconda/astroconda-dev'
+channel_URL: 'http://ssb.stsci.edu/astroconda-dev'
 numpy_version: 1.13
 packages:
   - costools
-  - crds
+  - cfitsio
+#  - ds9
+#  - crds
+#  - cube-tools
+#  - sphinxcontrib-programoutput
+#  - pyds9
+#  - wcstools
+#  - webbpsf
+#  
 #  - mosviz
 #  - specutils
 #  - glueviz