diff options
| -rw-r--r-- | jenkins/dispatch.groovy | 16 | ||||
| -rw-r--r-- | jenkins/generator_DSL.groovy | 10 | ||||
| -rw-r--r-- | jenkins/package_builder.groovy | 10 | ||||
| -rw-r--r-- | jenkins/scheduled_trigger.groovy | 10 | ||||
| -rw-r--r-- | manifests/dev-next.yaml | 103 | ||||
| -rw-r--r-- | manifests/dev.yaml | 1 | ||||
| -rw-r--r-- | manifests/jwst.yaml | 11 | ||||
| -rw-r--r-- | manifests/public-test.yaml | 1 | ||||
| -rw-r--r-- | manifests/public.yaml | 1 | ||||
| -rw-r--r-- | manifests/test-fail.yaml | 1 | ||||
| -rw-r--r-- | manifests/test.yaml | 1 |
11 files changed, 158 insertions, 7 deletions
diff --git a/jenkins/dispatch.groovy b/jenkins/dispatch.groovy index b537f7c..ed3639f 100644 --- a/jenkins/dispatch.groovy +++ b/jenkins/dispatch.groovy @@ -33,8 +33,18 @@ this.publication_lock_wait_s = 10 // Packages that appear in this file will be pinned to the version indicated. this.version_pins_file = "version_pins.yml" + node(LABEL) { + // Add any supplemental environment vars to the build environment. + for (env_var in this.supp_env_vars.trim().tokenize()) { + def key = env_var.tokenize("=")[0] + def val = env_var.tokenize("=")[1] + // env[] assignment requires in-process script approval for signature: + // org.codehaus.groovy.runtime.DefaultGroovyMethods putAt java.lang.Object + env[key] = val + } + // Delete any existing job workspace directory contents. // The directory deleted is the one named after the jenkins pipeline job. deleteDir() @@ -74,6 +84,7 @@ node(LABEL) { // Get the manifest and build control files git branch: BUILD_CONTROL_BRANCH, url: BUILD_CONTROL_REPO + // If a tag was specified in the job-suite-generator configuration, // explicitly check out that tag after cloning the (master) branch, // since the 'git' pipeline step does not yet support accessing tags. @@ -151,7 +162,7 @@ node(LABEL) { // Get conda recipes dir(this.recipes_dir) { - git url: this.manifest.repository + git branch: this.manifest.git_ref_spec, url: this.manifest.repository } // Get build utilities @@ -224,7 +235,7 @@ node(LABEL) { println("${this.version_pins_file} has no packages list, skipping pin environment creation.") this.use_version_pins = "false" } - + // (conda-build 3.x only) // Create and populate environment to be used for pinning reference when // building packages via the --bootstrap flag. Environment creation is done @@ -294,6 +305,7 @@ node(LABEL) { string(name: "cull_manifest", value: this.cull_manifest), string(name: "channel_URL", value: this.manifest.channel_URL), string(name: "use_version_pins", value: this.use_version_pins), + text(name: "supp_env_vars", value: this.supp_env_vars) ], propagate: false } diff --git a/jenkins/generator_DSL.groovy b/jenkins/generator_DSL.groovy index b9a3967..cace2f5 100644 --- a/jenkins/generator_DSL.groovy +++ b/jenkins/generator_DSL.groovy @@ -41,7 +41,6 @@ for (label in labels.trim().tokenize()) { //----------------------------------------------------------------------- // Create a folder to contain the jobs which are created below. - suite_name = "${manifest_file.tokenize(".")[0]}_${label}_py${py_version}_np${numpy_version}" folder(suite_name) { description("Build suite generated: ${job_def_generation_time}\n" + @@ -65,6 +64,10 @@ for (label in labels.trim().tokenize()) { "Whether or not package recipes that would generate a " + "package file name that already exists in the manfest's" + " channel archive are removed from the build list.") + textParam("supp_env_vars", + "", + "List of supplemental environment variables to define " + + "in the build envioronment.") } logRotator { numToKeep(this.num_builds_to_keep) @@ -108,7 +111,6 @@ for (label in labels.trim().tokenize()) { //----------------------------------------------------------------------- // Generate the series of actual package building jobs. - for(pkg in config.packages) { pipelineJob("${suite_name}/${pkg}") { @@ -153,6 +155,10 @@ for (label in labels.trim().tokenize()) { stringParam("use_version_pins", "false", "Whether or not to use global version pins.") + textParam("supp_env_vars", + "", + "List of supplemental environment variables to define " + + "in the build environment.") } definition { cps { diff --git a/jenkins/package_builder.groovy b/jenkins/package_builder.groovy index c4ec473..51ff9c8 100644 --- a/jenkins/package_builder.groovy +++ b/jenkins/package_builder.groovy @@ -6,6 +6,15 @@ this.build_status_file = "${this.parent_workspace}/propagated_build_status" node(this.label) { + // Add any supplemental environment vars to the build environment. + for (env_var in this.supp_env_vars.trim().tokenize()) { + def key = env_var.tokenize("=")[0] + def val = env_var.tokenize("=")[1] + // env[] assignment requires in-process script approval for signature: + // org.codehaus.groovy.runtime.DefaultGroovyMethods putAt java.lang.Object + env[key] = val + } + dir(this.parent_workspace) { env.PATH = "${this.parent_workspace}/miniconda/bin/:" + "${env.PATH}" @@ -40,6 +49,7 @@ node(this.label) { "parameter cull_manifest: ${this.cull_manifest}\n" + "parameter channel_URL: ${this.channel_URL}\n" + "parameter use_version_pins: ${this.use_version_pins}\n" + + "parameter supp_env_vars: ${this.supp_env_vars}\n" + "PATH: ${env.PATH}\n" + "PYTHONPATH: ${env.PYTHONPATH}\n" + "PYTHONUNBUFFERED: ${env.PYTHONUNBUFFERED}\n") diff --git a/jenkins/scheduled_trigger.groovy b/jenkins/scheduled_trigger.groovy index e8a3bdd..d1a5296 100644 --- a/jenkins/scheduled_trigger.groovy +++ b/jenkins/scheduled_trigger.groovy @@ -38,7 +38,7 @@ node("master") { // to parameters used to control the behavior of this job). Compose // the URL string used to pass those parameter values to the jobs // being triggered via the REST API. - params_url = "?" + params_url = "" params.each { key = it.key.toString() val = it.value.toString() @@ -47,7 +47,8 @@ node("master") { index = this.job_param_id.size() param_name = key[index..-1].trim() println(param_name) - params_url = "${params_url}${param_name}=${val}" + params_url = "${params_url}${param_name}=${val}\\&" + println("LOOP params_url: ${params_url}") } } @@ -62,8 +63,11 @@ node("master") { println("Platforms:\n${PLATFORMS}") for (platform in PLATFORMS.tokenize()) { println("Triggering _dispatch job for ${platform}...") + //trigger_url = "http://${url_base}/job/${abs_jobs_folder}/job/${platform}/" + + // "job/_dispatch/buildWithParameters${params_url} " + + // "-u ${USERNAME}:${PASSWORD}" trigger_url = "http://${url_base}/job/${abs_jobs_folder}/job/${platform}/" + - "job/_dispatch/buildWithParameters${params_url} " + + "job/_dispatch/buildWithParameters?${params_url} " + "-u ${USERNAME}:${PASSWORD}" println(trigger_url) sh (script: "curl -s -S -X POST -H ${CRUMB} ${trigger_url}") diff --git a/manifests/dev-next.yaml b/manifests/dev-next.yaml new file mode 100644 index 0000000..770ef00 --- /dev/null +++ b/manifests/dev-next.yaml @@ -0,0 +1,103 @@ +# Recipe repository +repository: 'https://github.com/jhunkeler/astroconda-dev' +git_ref_spec: 'next' + +# Publication channel to consult when determining what packages already exist. +channel_URL: 'http://ssb.stsci.edu/astroconda-dev' + +# publication_root path needs to be visible from the slave nodes. +publication_root: '/eng/ssb/websites/ssbpublic/astroconda-dev' +packages: + - acstools + - appdirs + - aprio + - asdf + - asdf-standard + - astroimtools + - astropy + - astroquery + - asv + - calcos + - cfitsio + - costools + - crds + - cubeviz + - d2to1 + - dbus-glib + - dbus-python + - drizzle + - drizzlepac + - ds9 + - fftw + - fitsblender + - ginga + - glue-core + - glue-vispy-viewers + - glueviz + - gwcs + - hstcal + - htc_utils + - imexam + - jplephem + - jwst + - jwst_coronagraph_visibility + - jwst_gtvt + - jwxml + - mosviz + - namedlist + - nictools + - opuscoords + - photutils + - poppy + - purge_path + - py-expression-eval + - pyds9 + - pyfftw + - pymssql + - pyobjc-core + - pyobjc-framework-cocoa + - pyobjc-framework-quartz + - pyqtgraph + - pyraf + - pyregion + - pysynphot + - python-daemon + - pytools + - pywcs + - reftools + - relic + - secretstorage + - selenium + - shunit2 + - specutils + - specviz + - spherical-geometry + - sphinx_rtd_theme + - sphinxcontrib-programoutput + - stginga + - stistools + - stsci + - stsci-data-analysis + - stsci-hst + - stsci.convolve + - stsci.distutils + - stsci.image + - stsci.imagemanip + - stsci.imagestats + - stsci.ndimage + - stsci.numdisplay + - stsci.skypac + - stsci.sphere + - stsci.sphinxext + - stsci.stimage + - stsci.tools + - stsynphot + - stwcs + - synphot + - verhawk + - wcstools + - webbpsf + - webbpsf-data + - wfc3tools + - wfpc2tools + - xpa diff --git a/manifests/dev.yaml b/manifests/dev.yaml index cbed34a..68e799b 100644 --- a/manifests/dev.yaml +++ b/manifests/dev.yaml @@ -1,5 +1,6 @@ # Recipe repository repository: 'https://github.com/astroconda/astroconda-dev' +git_ref_spec: 'master' # Publication channel to consult when determining what packages already exist. channel_URL: 'http://ssb.stsci.edu/astroconda-dev' diff --git a/manifests/jwst.yaml b/manifests/jwst.yaml new file mode 100644 index 0000000..adf14c5 --- /dev/null +++ b/manifests/jwst.yaml @@ -0,0 +1,11 @@ +# Recipe repository +repository: 'https://github.com/rendinam/astroconda-dev' +git_ref_spec: 'jwst_parameterized_rev' + +# Publication channel to consult when determining what packages already exist. +channel_URL: 'http://ssb.stsci.edu/astroconda-dev' + +# publication_root path needs to be visible from the slave nodes. +publication_root: '/eng/ssb/websites/ssbpublic/astroconda-j-dev-testing' +packages: + - jwst diff --git a/manifests/public-test.yaml b/manifests/public-test.yaml index 331d857..901c39e 100644 --- a/manifests/public-test.yaml +++ b/manifests/public-test.yaml @@ -1,5 +1,6 @@ # Recipe repository repository: 'https://github.com/astroconda/astroconda-contrib' +git_ref_spec: 'master' # Publication channel to consult when determining what packages already exist. channel_URL: 'http://ssb.stsci.edu/astroconda' diff --git a/manifests/public.yaml b/manifests/public.yaml index 084056e..151bd5a 100644 --- a/manifests/public.yaml +++ b/manifests/public.yaml @@ -1,5 +1,6 @@ # Recipe repository repository: 'https://github.com/astroconda/astroconda-contrib' +git_ref_spec: 'master' # Publication channel to consult when determining what packages already exist. channel_URL: 'http://ssb.stsci.edu/astroconda' diff --git a/manifests/test-fail.yaml b/manifests/test-fail.yaml index a2d9265..055ef8f 100644 --- a/manifests/test-fail.yaml +++ b/manifests/test-fail.yaml @@ -1,5 +1,6 @@ # Recipe repository repository: 'https://github.com/astroconda/astroconda-contrib' +git_ref_spec: 'master' # Publication channel to consult when determining what packages already exist. channel_URL: 'http://ssb.stsci.edu/astroconda' diff --git a/manifests/test.yaml b/manifests/test.yaml index 3dbbf79..5bc0a71 100644 --- a/manifests/test.yaml +++ b/manifests/test.yaml @@ -1,5 +1,6 @@ # Recipe repository repository: 'https://github.com/astroconda/astroconda-contrib' +git_ref_spec: 'master' # Publication channel to consult when determining what packages already exist. channel_URL: 'http://ssb.stsci.edu/astroconda' |