diff options
Diffstat (limited to 'jenkins')
| -rw-r--r-- | jenkins/dispatch.groovy | 80 | ||||
| -rw-r--r-- | jenkins/generator_DSL.groovy | 66 | ||||
| -rw-r--r-- | jenkins/job-suite-generator.groovy | 20 |
3 files changed, 99 insertions, 67 deletions
diff --git a/jenkins/dispatch.groovy b/jenkins/dispatch.groovy index 227d1fd..4ba44fd 100644 --- a/jenkins/dispatch.groovy +++ b/jenkins/dispatch.groovy @@ -1,27 +1,24 @@ -// The conda version shown in the conda_installers list below is installed -// Where to obtain this file and the manifest files -this.build_control_URL = "https://github.com/astroconda/build_control" - -// first, then the version is forced to this value. -this.conda_version = "4.2.15" - -// Conda-build is installed fresh at this version. -this.conda_build_version = "2.1.1" - -// Where to get the conda installer -this.conda_base_URL = "https://repo.continuum.io/miniconda/" - -this.recipes_dir = "conda-recipes" - -// Support utilities -this.utils_URL = "https://github.com/rendinam/rambo" +// Parameters inherited from the calling script via environment injection. +//---------------------------------------------------------------------------- +// MANIFEST_FILE - The "release" type; list of recipes/packages to build +// LABEL - Node or logical group of build nodes +// PY_VERSION - Python version hosted by conda to support the build +// BUILD_CONTROL_REPO - Repository holding this & other build system files, +// and manifest files +// CONDA_VERSION - First, then the version is forced to this value. +// CONDA_BUILD_VERSION - Conda-build is installed forced to this version. +// CONDA_BASE_URL - Where to get the conda installer +// UTILS_REPO - Repository holding support utilities + +// Directories to create within the workspace this.utils_dir = "utils" +this.recipes_dir = "conda-recipes" // The conda installer script to use for various <OS><py_version> combinations. -this.conda_installers = ["Linux-py2.7":"Miniconda2-4.2.12-Linux-x86_64.sh", - "Linux-py3.5":"Miniconda3-4.2.12-Linux-x86_64.sh", - "MacOSX-py2.7":"Miniconda2-4.2.12-MacOSX-x86_64.sh", - "MacOSX-py3.5":"Miniconda3-4.2.12-MacOSX-x86_64.sh"] +this.conda_installers = ["Linux-py2.7":"Miniconda2-${CONDA_VERSION}-Linux-x86_64.sh", + "Linux-py3.5":"Miniconda3-${CONDA_VERSION}-Linux-x86_64.sh", + "MacOSX-py2.7":"Miniconda2-${CONDA_VERSION}-MacOSX-x86_64.sh", + "MacOSX-py3.5":"Miniconda3-${CONDA_VERSION}-MacOSX-x86_64.sh"] node(LABEL) { @@ -68,10 +65,17 @@ node(LABEL) { assert MANIFEST_FILE != null assert MANIFEST_FILE != "manifest_file-DEFAULTVALUE" + if (CONDA_BASE_URL[-1..-1] == "/") { + CONDA_BASE_URL = [0..-2] + } + + // Parameters passed at trigger time + println("this.cull_manifest: ${this.cull_manifest}") + println("PATH = ${env.PATH}") // Fetch the manifest files - git url: this.build_control_URL + git url: BUILD_CONTROL_REPO // Check for the availability of a download tool and then use it // to get the conda installer. @@ -93,14 +97,14 @@ node(LABEL) { def conda_installer = this.conda_installers["${this.OSname}-py${PY_VERSION}"] - dl_cmd = dl_cmd + " ${this.conda_base_URL}${conda_installer}" + dl_cmd = dl_cmd + " ${CONDA_BASE_URL}/${conda_installer}" sh dl_cmd // Run miniconda installer and then force to particular version sh "bash ./${conda_installer} -b -p miniconda" env.PATH = "${env.WORKSPACE}/miniconda/bin/:${env.PATH}" - sh "conda install --quiet conda=${this.conda_version}" - sh "conda install --quiet --yes conda-build=${this.conda_build_version}" + sh "conda install --quiet conda=${CONDA_VERSION}" + sh "conda install --quiet --yes conda-build=${CONDA_BUILD_VERSION}" // Apply bugfix patch to conda_build 2.1.1 def patches_dir = "${env.WORKSPACE}/patches" @@ -109,9 +113,8 @@ node(LABEL) { sh "patch ${patch}" } - this.manifest = readYaml file: "manifests/" + - this.manifest_file - if (this.manifest.channel_URL == '/') { + this.manifest = readYaml file: "manifests/${MANIFEST_FILE}" + if (this.manifest.channel_URL[-1..-1] == "/") { this.manifest.channel_URL = this.manifest.channel_URL[0..-2] } println("Manifest repository: ${this.manifest.repository}") @@ -132,17 +135,22 @@ node(LABEL) { stage("Generate build list") { // Obtain build utilities dir(this.utils_dir) { - git url: this.utils_URL + git url: UTILS_REPO } - // Generate a filtered, culled, & dependency-ordered list of available - // package recipes. + // Generate a filtered, optionally culled, & dependency-ordered list + // of available package recipes. + def culled_option = "--culled" + if (this.cull_manifest == "false") { + culled_option = "" + } def blist_file = "build_list" cmd = "${this.utils_dir}/rambo.py" args = ["--platform ${this.CONDA_PLATFORM}", - "--manifest manifests/${this.manifest_file}", + "--python ${PY_VERSION}", + "--manifest manifests/${MANIFEST_FILE}", "--file ${blist_file}", - "--culled", + "${culled_option}", this.recipes_dir] for (arg in args) { cmd = "${cmd} ${arg}" @@ -150,13 +158,13 @@ node(LABEL) { sh(script: cmd) def blist_text = readFile blist_file - def build_list = blist_text.trim().tokenize() + this.build_list = blist_text.trim().tokenize() println("Build list:") - println(build_list) + println(this.build_list) } stage("Build packages") { - for (pkg in build_list) { + for (pkg in this.build_list) { build job: pkg, parameters: [string(name: "label", value: env.NODE_NAME), diff --git a/jenkins/generator_DSL.groovy b/jenkins/generator_DSL.groovy index 1a77119..a4b40a8 100644 --- a/jenkins/generator_DSL.groovy +++ b/jenkins/generator_DSL.groovy @@ -10,7 +10,7 @@ def config = yaml.load(readFileFromWorkspace("manifests/${manifest_file}")) //----------------------------------------------------------------------- // Create a folder to contain the jobs which are created below. -suite_name = "${manifest_file.tokenize('.')[0]}_${label}_py${py_version}" +suite_name = "${manifest_file.tokenize(".")[0]}_${label}_py${py_version}" folder(suite_name) @@ -19,17 +19,35 @@ folder(suite_name) // build jobs. pipelineJob("${suite_name}/_dispatch") { - println("label = ${label}") - println("manifest_file = ${manifest_file}") - println("py_version = ${py_version}") + // At trigger-time, allow for setting manifest culling behavior. + parameters { + booleanParam("cull_manifest", + true, + "Whether or not package recipes that would generate a " + + "package file name that already exists in the manfest's" + + " channel archive are removed from the build list.") + } + println("MANIFEST_FILE = ${manifest_file}") + println("LABEL = ${label}") + println("PY_VERSION = ${py_version}") + println("BUILD_CONTROL_REPO = ${build_control_repo}") + println("CONDA_VERSION = ${conda_version}") + println("CONDA_BUILD_VERSION = ${conda_build_version}") + println("CONDA_BASE_URL = ${conda_base_URL}") + println("UTILS_REPO = ${utils_repo}") environmentVariables { - env("LABEL", "${label}") - env("MANIFEST_FILE", "${manifest_file}") - env("PY_VERSION", "${py_version}") + env("MANIFEST_FILE", manifest_file) + env("LABEL", label) + env("PY_VERSION", py_version) + env("BUILD_CONTROL_REPO", build_control_repo) + env("CONDA_VERSION", conda_version) + env("CONDA_BUILD_VERSION", conda_build_version) + env("CONDA_BASE_URL", conda_base_URL) + env("UTILS_REPO", utils_repo) } definition { cps { - script(readFileFromWorkspace('jenkins/dispatch.groovy')) + script(readFileFromWorkspace("jenkins/dispatch.groovy")) sandbox() } } @@ -43,25 +61,25 @@ for(pkg in config.packages) { pipelineJob("${suite_name}/${pkg}") { parameters { - stringParam('label', - 'label-DEFAULTVALUE', - 'The node on which to run.') - stringParam('py_version', - 'py_version-DEFAULTVALUE', - 'python version to use') - stringParam('numpy_version', - 'numpy_version-DEFAULTVALUE', - 'Version of numpy to use') - stringParam('parent_workspace', - 'parent_workspace-DEFAULTVALUE', - 'The workspace dir of the dispatch job') - stringParam('manifest_file', - 'manifest_file-DEFAULTVALUE', - 'Manifest (release) file to use for the build.') + stringParam("label", + "label-DEFAULTVALUE", + "The node on which to run.") + stringParam("py_version", + "py_version-DEFAULTVALUE", + "python version to use") + stringParam("numpy_version", + "numpy_version-DEFAULTVALUE", + "Version of numpy to use") + stringParam("parent_workspace", + "parent_workspace-DEFAULTVALUE", + "The workspace dir of the dispatch job") + stringParam("manifest_file", + "manifest_file-DEFAULTVALUE", + "Manifest (release) file to use for the build.") } definition { cps { - script(readFileFromWorkspace('jenkins/package_builder.groovy')) + script(readFileFromWorkspace("jenkins/package_builder.groovy")) sandbox() } } diff --git a/jenkins/job-suite-generator.groovy b/jenkins/job-suite-generator.groovy index 9130d37..9ef3e08 100644 --- a/jenkins/job-suite-generator.groovy +++ b/jenkins/job-suite-generator.groovy @@ -16,10 +16,16 @@ node("master") { // These variables are provided by the execution of the generator // build task with parameters. Each var is populated by a parameter // specification. - sh "echo manifest_file=${this.manifest_file}" - sh "echo label=${this.label}" - sh "echo py_version=${this.py_version}" - sh "echo old_jobs_action=${this.old_jobs_action}" + println("manifest_file=${this.manifest_file}") + println("label=${this.label}") + println("py_version=${this.py_version}") + println("build_control_repo=${this.build_control_repo}") + println("conda_version=${this.conda_version}") + println("conda_build_version=${this.conda_build_version}") + println("conda_base_URL=${this.conda_base_URL}") + println("utils_repo=${this.utils_repo}") + + println("old_jobs_action=${this.old_jobs_action}") } stage("Setup") { @@ -34,10 +40,10 @@ node("master") { sh "cp -r ${env.WORKSPACE}@script/* ." } - stage('Spawn job definitions') { - jobDsl targets: ["jenkins/generator_DSL.groovy"].join('\n'), + stage("Spawn job definitions") { + jobDsl targets: ["jenkins/generator_DSL.groovy"].join("\n"), lookupStrategy: "SEED_JOB", - additionalClasspath: ["${this.ldir}/*.jar"].join('\n'), + additionalClasspath: ["${this.ldir}/*.jar"].join("\n"), removeAction: "${this.old_jobs_action}" } |