diff options
| -rw-r--r-- | jenkins/dispatch.groovy | 157 | ||||
| -rw-r--r-- | jenkins/generator_DSL.groovy | 71 | ||||
| -rw-r--r-- | jenkins/job-suite-generator.groovy | 45 | ||||
| -rw-r--r-- | jenkins/package_builder.groovy | 68 |
4 files changed, 341 insertions, 0 deletions
diff --git a/jenkins/dispatch.groovy b/jenkins/dispatch.groovy new file mode 100644 index 0000000..77ceb36 --- /dev/null +++ b/jenkins/dispatch.groovy @@ -0,0 +1,157 @@ +// The conda version shown in the conda_installers list below is installed +// Where to obtain this file and the manifest files +this.build_control_URL = "https://github.com/astroconda/build_control" + +// first, then the version is forced to this value. +this.conda_version = "4.2.15" + +// Conda-build is installed fresh at this version. +this.conda_build_version = "2.1.1" + +// Where to get the conda installer +this.conda_base_URL = "https://repo.continuum.io/miniconda/" + +this.recipes_dir = "conda-recipes" + +// Support utilities +this.utils_URL = "https://github.com/rendinam/rambo" +this.utils_dir = "utils" + +// The conda installer script to use for various <OS><py_version> combinations. +this.conda_installers = ["Linux-py2.7":"Miniconda2-4.2.12-Linux-x86_64.sh", + "Linux-py3.5":"Miniconda3-4.2.12-Linux-x86_64.sh", + "MacOSX-py2.7":"Miniconda2-4.2.12-MacOSX-x86_64.sh", + "MacOSX-py3.5":"Miniconda3-4.2.12-MacOSX-x86_64.sh"] + +node(LABEL) { + + this.OSname = null + def uname = sh(script: "uname", returnStdout: true).trim() + if (uname == "Darwin") { + this.OSname = "MacOSX" + env.PATH = "${env.PATH}:/sw/bin" + this.CONDA_BLD_OUTPUT_DIR = "osx-64" + } + if (uname == "Linux") { + this.OSname = uname + this.CONDA_BLD_OUTPUT_DIR = "linux-64" + } + assert uname != null + + println("NODE_NAME = ${env.NODE_NAME}") + + // Delete any existing job workspace directory contents. + // The directory deleted is the one named after the jenkins pipeline job. + deleteDir() + + // Allow for sharing build_list between stages below. + this.build_list = [] + + stage('Setup') { + + // Inherited from env() assignment performed in the generator + // DSL script. + println "LABEL = ${LABEL}" + assert LABEL != null + assert LABEL != "label-DEFAULTVALUE" + + // Inherited from env() assignment performed in the generator + // DSL script. + println("PY_VERSION = ${PY_VERSION}") + assert PY_VERSION != null + assert PY_VERSION != "py_version-DEFAULTVALUE" + + // Inherited from env() assignment performed in the generator + // DSL script. + println("MANIFEST_FILE = ${MANIFEST_FILE}") + assert MANIFEST_FILE != null + assert MANIFEST_FILE != "manifest_file-DEFAULTVALUE" + + // Fetch the manifest files + git url: this.build_control_URL + + // Check for the availability of a download tool and then use it + // to get the conda installer. + def dl_cmds = ["wget --no-verbose --server-response --no-check-certificate", + "curl -OSs"] + def dl_cmd = null + def stat1 = 999 + for (cmd in dl_cmds) { + stat1 = sh(script: "which ${cmd.split()[0]}", returnStatus: true) + if( stat1 == 0 ) { + dl_cmd = cmd + break + } + } + if (stat1 != 0) { + println("Could not find a download tool. Unable to proceed.") + sh "false" + } + + def conda_installer = + this.conda_installers["${this.OSname}-py${PY_VERSION}"] + dl_cmd = dl_cmd + " ${this.conda_base_URL}${conda_installer}" + sh dl_cmd + + // Run miniconda installer and then force to particular version + sh "bash ./${conda_installer} -b -p miniconda" + env.PATH = "${env.WORKSPACE}/miniconda/bin/:" + "${env.PATH}" + sh "conda install --quiet conda=${this.conda_version}" + sh "conda install --quiet --yes conda-build=${this.conda_build_version}" + + this.manifest = readYaml file: "manifests/" + + this.manifest_file + println("Manifest repository: ${this.manifest.repository}") + println("Manifest numpy version specification: " + + "${this.manifest.numpy_version}") + println("Manifest packages to build:") + for (pkgname in this.manifest.packages) { + println(pkgname) + } + + // Retrieve conda recipes + dir(this.recipes_dir) { + git url: this.manifest.repository + } + + // Retrieve recipe management tools + + + } + + stage("Generate build list") { + // Call in Rambo. + dir(this.utils_dir) { + git url: this.utils_URL + } + + // Generate a dependency-ordered list of available package recipes. + cmd = "${this.utils_dir}/rambo.py --ordered ${this.recipes_dir}" + ordered_available = + sh(script: cmd, returnStdout: true).trim().tokenize() + + // Compose the ordered union of the list of available recipes and the + // actual build manifest. + build_list = [] + for (pkg in ordered_available) { + if (pkg in this.manifest.packages) { + build_list.push(pkg) + } + } + println("Build list:") + println(build_list) + } + + stage('Build packages') { + for (pkg in build_list) { + build job: pkg, + parameters: + [string(name: 'label', value: env.NODE_NAME), + string(name: 'py_version', value: PY_VERSION), + string(name: 'numpy_version', value: "${this.manifest.numpy_version}"), + string(name: 'parent_workspace', value: env.WORKSPACE)], + propagate: false + } + } +} + diff --git a/jenkins/generator_DSL.groovy b/jenkins/generator_DSL.groovy new file mode 100644 index 0000000..5f5d701 --- /dev/null +++ b/jenkins/generator_DSL.groovy @@ -0,0 +1,71 @@ +// Job generator script. Uses Job-DSL plugin API. + +// Third party YAML parsing class. Obtain from URL below before use. +// https://repo1.maven.org/maven2/org/yaml/snakeyaml/1.17/snakeyaml-1.17.jar +import org.yaml.snakeyaml.Yaml + +def yaml = new Yaml() +def config = yaml.load(readFileFromWorkspace("manifests/${manifest_file}")) + + +//----------------------------------------------------------------------- +// Create a folder to contain the jobs which are created below. +suite_name = "${manifest_file.tokenize('.')[0]}_${label}_py${py_version}" +folder(suite_name) + + +//----------------------------------------------------------------------- +// Generate the dispatch job that will trigger the chain of package +// build jobs. + +pipelineJob("${suite_name}/dispatch") { + println("label = ${label}") + println("manifest_file = ${manifest_file}") + println("py_version = ${py_version}") + environmentVariables { + env("LABEL", "${label}") + env("MANIFEST_FILE", "${manifest_file}") + env("PY_VERSION", "${py_version}") + } + definition { + cps { + script(readFileFromWorkspace('jenkins/dispatch.groovy')) + sandbox() + } + } +} + + +//----------------------------------------------------------------------- +// Generate the series of actual package building jobs. + +for(pkg in config.packages) { + + pipelineJob("${suite_name}/${pkg}") { + parameters { + stringParam('label', + 'label-DEFAULTVALUE', + 'The node on which to run.') + stringParam('py_version', + 'py_version-DEFAULTVALUE', + 'python version to use') + stringParam('numpy_version', + 'numpy_version-DEFAULTVALUE', + 'Version of numpy to use') + stringParam('parent_workspace', + 'parent_workspace-DEFAULTVALUE', + 'The workspace dir of the dispatch job') + stringParam('manifest_file', + 'manifest_file-DEFAULTVALUE', + 'Manifest (release) file to use for the build.') + } + definition { + cps { + script(readFileFromWorkspace('jenkins/package_builder.groovy')) + sandbox() + } + } + } // end pipelineJob + +} //end for(pkg... + diff --git a/jenkins/job-suite-generator.groovy b/jenkins/job-suite-generator.groovy new file mode 100644 index 0000000..9130d37 --- /dev/null +++ b/jenkins/job-suite-generator.groovy @@ -0,0 +1,45 @@ +// Top-level pipeline job that provides parameterized machinery for +// creating one or more build job suites for use in building AstroConda +// package sets. +// Uses Job-DSL plugin. + +// Directory into which supporting libraries are stored. Gets added to +// groovy classpath definition prior to imports. +this.ldir = "libs" + +node("master") { + + stage("Prep") { + // Delete any existing job workspace directory contents. + deleteDir() + + // These variables are provided by the execution of the generator + // build task with parameters. Each var is populated by a parameter + // specification. + sh "echo manifest_file=${this.manifest_file}" + sh "echo label=${this.label}" + sh "echo py_version=${this.py_version}" + sh "echo old_jobs_action=${this.old_jobs_action}" + } + + stage("Setup") { + sh "mkdir -p ${this.ldir}" + // Obtain libraries to facilitate job generation tasks. + dir ("libs") { + sh "curl -O https://repo1.maven.org/maven2/org/yaml/snakeyaml/1.17/snakeyaml-1.17.jar" + } + // Copy files from the implicit checkout of the build_control directory (handled by + // the job that reads this pipeline script) into the actual workspace of this job so + // the jobDsl call below will be able to find what it needs. + sh "cp -r ${env.WORKSPACE}@script/* ." + } + + stage('Spawn job definitions') { + jobDsl targets: ["jenkins/generator_DSL.groovy"].join('\n'), + lookupStrategy: "SEED_JOB", + additionalClasspath: ["${this.ldir}/*.jar"].join('\n'), + removeAction: "${this.old_jobs_action}" + } + +} + diff --git a/jenkins/package_builder.groovy b/jenkins/package_builder.groovy new file mode 100644 index 0000000..2391abf --- /dev/null +++ b/jenkins/package_builder.groovy @@ -0,0 +1,68 @@ +node(this.label) { + + dir(this.parent_workspace) { + + println("inherited workspace: ${this.parent_workspace}") + println("Nodelabel: ${this.label}") + println("${env.JOB_NAME}") + println("${env.JOB_BASE_NAME}") + println("${env.BUILD_NUMBER}") + println("${env.NODE_NAME}") + println("${env.WORKSPACE}") + println("${env.JENKINS_HOME}") + println(currentBuild.buildVariables) + println("parameter py_version: ${this.py_version}") + + env.PATH = "${this.parent_workspace}/miniconda/bin/:" + "${env.PATH}" + + // Make the log files a bit more deterministic + env.PYTHONUNBUFFERED = "true" + + this.OSname = null + uname = sh(script: "uname", returnStdout: true).trim() + if (uname == "Darwin") { + this.OSname = "MacOSX" + env.PATH = "${env.PATH}:/sw/bin" + this.CONDA_BLD_OUTPUT_DIR = "osx-64" + } + if (uname == "Linux") { + this.OSname = uname + this.CONDA_BLD_OUTPUT_DIR = "linux-64" + } + assert uname != null + println("${this.CONDA_BLD_OUTPUT_DIR}") + + // In directory common to all package build jobs, run conda build for this + // package. + dir("conda-recipes") { + + build_cmd = "conda build" + + stage("Build") { + build_args = "--no-test --no-anaconda-upload --python=${this.py_version}" + + " --numpy=${this.numpy_version} --skip-existing" + stat = 999 + + stat = sh(script: "${build_cmd} ${build_args} ${env.JOB_BASE_NAME}", + returnStatus: true) + println("Shell call returned status: ${stat}") + if (stat != 0) { + currentBuild.result = "FAILURE" + } + } + + stage("Test") { + build_args = "--test --no-anaconda-upload --python=${this.py_version}" + + " --numpy=${this.numpy_version} --skip-existing" + stat = sh(script: "${build_cmd} ${build_args} ${env.JOB_BASE_NAME}", + returnStatus: true) + println("Shell call returned status: ${stat}") + if (stat != 0) { + currentBuild.result = "UNSTABLE" + } + } + + } // end dir + } + +} //end node |