From 491446f64369d5b7034489371a82a301631692ba Mon Sep 17 00:00:00 2001 From: Matt Rendina Date: Tue, 6 Jun 2017 15:06:31 -0400 Subject: Testing channel use for culled manifests --- jenkins/astroconda_parameterized.groovy | 212 ----------------------- jenkins/dispatch.groovy | 6 +- jenkins/obsolete/astroconda_parameterized.groovy | 212 +++++++++++++++++++++++ jenkins/package_builder.groovy | 15 +- 4 files changed, 230 insertions(+), 215 deletions(-) delete mode 100644 jenkins/astroconda_parameterized.groovy create mode 100644 jenkins/obsolete/astroconda_parameterized.groovy diff --git a/jenkins/astroconda_parameterized.groovy b/jenkins/astroconda_parameterized.groovy deleted file mode 100644 index 67932ae..0000000 --- a/jenkins/astroconda_parameterized.groovy +++ /dev/null @@ -1,212 +0,0 @@ -// Astroconda build control script for use with the Jenkins CI system -//---------------------------------------------------------------------------- -// This script is typically dispatched from a multi-configuration (build -// matrix) job. It accepts the following parameters controlling its behavior -// at dispatch time: -// -// label - The build node on which to execute -// ("nott", "boyle", etc) -// release_type - Determines which manifest to use -// ("dev", "public", "test", etc) -// py_version - The version of python to build upon (ships with conda) -// ("py2.7", "py3.5", ...) -// -// Constants controlling the behavior of this script: -// -// Where to obtain this file and the manifest files -this.build_control_URL = "https://github.com/astroconda/build_control" - -// The conda version shown in the conda_installers list below is installed -// first, then the version is forced to this value. -this.conda_version = "4.2.15" - -// Conda-build is installed fresh at this version. -this.conda_build_version = "2.1.1" - -// Where to get the conda installer -this.conda_base_URL = "https://repo.continuum.io/miniconda/" - -// The conda installer script to use for various combinations. -this.conda_installers = ["Linux-py2.7":"Miniconda2-4.2.12-Linux-x86_64.sh", - "Linux-py3.5":"Miniconda3-4.2.12-Linux-x86_64.sh", - "MacOSX-py2.7":"Miniconda2-4.2.12-MacOSX-x86_64.sh", - "MacOSX-py3.5":"Miniconda3-4.2.12-MacOSX-x86_64.sh"] - -// The manifest file to use for each release type -this.manifest_files = ["dev":"dev.yaml", - "public":"public.yaml", - "public_legacy":"public_legacy.yaml", - "ETC":"etc.yaml", - "dev_rotate_meta_packages":"astroconda.dev_rotate.yaml", - "test":"test.yaml"] -//---------------------------------------------------------------------------- - -node(this.label) { - - this.OSname = null - def uname = sh(script: "uname", returnStdout: true).trim() - if (uname == "Darwin") { - this.OSname = "MacOSX" - env.PATH = "${env.PATH}:/sw/bin" - this.CONDA_BLD_OUTPUT_DIR = "osx-64" - } - if (uname == "Linux") { - this.OSname = uname - this.CONDA_BLD_OUTPUT_DIR = "linux-64" - } - assert uname != null - - this.BUILD_SUBDIR = "${this.release_type}/${this.py_version}" - this.BUILD_ROOT = "${env.WORKSPACE}/${this.BUILD_SUBDIR}" - - // Delete any existing job workspace directory contents. - // The directory deleted is the one named after the jenkins pipeline job. - deleteDir() - - // Perform the following build job tasks in BUILD_SUBDIR within the - // topmost workspace directory for this node to allow for multiple - // parallel jobs using different combinations of build parameters. - dir("${this.BUILD_SUBDIR}") { - - stage('Setup') { - - println "this.label ${this.label}" - assert this.label != null - assert this.label != "label-DEFAULT" - - println "this.py_version ${this.py_version}" - assert this.py_version != null - assert this.py_version != "py_version-DEFAULT" - - println "this.release_type ${this.release_type}" - assert this.release_type != null - assert this.release_type != "release_type-DEFAULT" - - // Fetch the manifest files - git url: this.build_control_URL - - // Check for the availability of a download tool and then use it - // to get the conda installer. - def dl_cmds = ["wget --no-verbose --server-response --no-check-certificate", - "curl -OSs"] - def dl_cmd = null - def stat1 = 999 - for (cmd in dl_cmds) { - stat1 = sh(script: "which ${cmd.split()[0]}", returnStatus: true) - if( stat1 == 0 ) { - dl_cmd = cmd - break - } - } - if (stat1 != 0) { - println("Could not find a download tool. Unable to proceed.") - sh "false" - } - - def conda_installer = - this.conda_installers["${this.OSname}-${this.py_version}"] - dl_cmd = dl_cmd + " ${this.conda_base_URL}${conda_installer}" - sh dl_cmd - - // Make the log files a bit more deterministic - env.PYTHONUNBUFFERED = "true" - - // Run miniconda installer and then force to particular version - sh "bash ./${conda_installer} -b -p miniconda" - env.PATH = "${this.BUILD_ROOT}/miniconda/bin/:" + "${env.PATH}" - sh "conda install --quiet conda=${this.conda_version}" - sh "conda install --quiet --yes conda-build=${this.conda_build_version}" - - // TODO: Check for presence of build support tools, such as git, - // make, compilers, etc. and print a summary of their versions - // for auditing purposes. - - this.manifest = readYaml file: "manifests/" + - this.manifest_files["${this.release_type}"] - println("Manifest repository: ${this.manifest.repository}") - println("Manifest numpy version specification: " + - "${this.manifest.numpy_version}") - println("Manifest packages to build:") - for (pkgname in this.manifest.packages) { - println(pkgname) - } - - println("Checking for supplemental channels specification:") - if (this.manifest.channels != null) { - println("Supplemental channel(s) found, adding:") - for (channel in this.manifest.channels) { - sh "conda config --add channels ${channel}" - } - } else { - println("[INFO] No supplemental channels specified in manifest.") - } - - // Retrieve conda recipes - def recipes_dir = "conda-recipes" - dir(recipes_dir) { - git url: this.manifest.repository - } - - // Set git ID info to avoid errors with commands like 'git tag'. - sh "git config user.email \"ssb-jenkins@ssb-jenkins\"" - sh "git config user.name \"ssb-jenkins\"" - } - - stage('Build packages') { - def build_cmd = "conda build" - def pyversion_num = this.py_version.split('y')[1] - def build_args = "--no-anaconda-upload --python=${pyversion_num}" + - " --numpy=${this.manifest.numpy_version} --skip-existing" - def stat2 = 999 - dir ("conda-recipes") { - for (pkgname in this.manifest.packages) { - stat2 = sh(script: "${build_cmd} ${build_args} ${pkgname}", - returnStatus: true) - println("Shell call returned status: ${stat2}") - println "\nEnd Build: ${pkgname} " + - "=======================================================" - } - } - - // Determine if each package in the manifest was built. - // Set flag to fail the stage in the event of any missing packages. - def stage_success = "true" - def outdir = "miniconda/conda-bld/${this.CONDA_BLD_OUTPUT_DIR}" - dir(outdir) { - def built_pkgs = sh(script: "ls -1 *.tar.bz2", - returnStdout: true).trim().split() - def found_pkg = false - def first = true - for (pkgname in this.manifest.packages) { - found_pkg = false - for (built_pkg in built_pkgs) { - if ( built_pkg.indexOf(pkgname, 0) != -1) { - found_pkg = true - break - } - } - if (!found_pkg) { - if (first) { - println("ERROR BUILDING ONE OR MORE PACKAGES!\n") - first = false - } - println("ERROR building: ${pkgname}") - stage_success = "false" - } - } - } - sh stage_success - } - - stage('Tests') { - println "Test results go here." - } - - stage('Publish/Archive build products') { - // Publishing and/or archival steps go here. - } - - } // end dir() - -} //end node - diff --git a/jenkins/dispatch.groovy b/jenkins/dispatch.groovy index d671f28..f05d906 100644 --- a/jenkins/dispatch.groovy +++ b/jenkins/dispatch.groovy @@ -177,7 +177,7 @@ node(LABEL) { // of available package recipes. def culled_option = "--culled" if (this.cull_manifest == "false") { - culled_option = "" + culled_option = "" } def build_list_file = "build_list" cmd = "rambo" @@ -205,7 +205,9 @@ node(LABEL) { string(name: "py_version", value: PY_VERSION), string(name: "numpy_version", value: "${this.manifest.numpy_version}"), - string(name: "parent_workspace", value: env.WORKSPACE)], + string(name: "parent_workspace", value: env.WORKSPACE), + string(name: "cull_manifest", value: this.cull_manifest), + string(name: "channel_URL", value: this.manifest.channel_URL)], propagate: false } } diff --git a/jenkins/obsolete/astroconda_parameterized.groovy b/jenkins/obsolete/astroconda_parameterized.groovy new file mode 100644 index 0000000..67932ae --- /dev/null +++ b/jenkins/obsolete/astroconda_parameterized.groovy @@ -0,0 +1,212 @@ +// Astroconda build control script for use with the Jenkins CI system +//---------------------------------------------------------------------------- +// This script is typically dispatched from a multi-configuration (build +// matrix) job. It accepts the following parameters controlling its behavior +// at dispatch time: +// +// label - The build node on which to execute +// ("nott", "boyle", etc) +// release_type - Determines which manifest to use +// ("dev", "public", "test", etc) +// py_version - The version of python to build upon (ships with conda) +// ("py2.7", "py3.5", ...) +// +// Constants controlling the behavior of this script: +// +// Where to obtain this file and the manifest files +this.build_control_URL = "https://github.com/astroconda/build_control" + +// The conda version shown in the conda_installers list below is installed +// first, then the version is forced to this value. +this.conda_version = "4.2.15" + +// Conda-build is installed fresh at this version. +this.conda_build_version = "2.1.1" + +// Where to get the conda installer +this.conda_base_URL = "https://repo.continuum.io/miniconda/" + +// The conda installer script to use for various combinations. +this.conda_installers = ["Linux-py2.7":"Miniconda2-4.2.12-Linux-x86_64.sh", + "Linux-py3.5":"Miniconda3-4.2.12-Linux-x86_64.sh", + "MacOSX-py2.7":"Miniconda2-4.2.12-MacOSX-x86_64.sh", + "MacOSX-py3.5":"Miniconda3-4.2.12-MacOSX-x86_64.sh"] + +// The manifest file to use for each release type +this.manifest_files = ["dev":"dev.yaml", + "public":"public.yaml", + "public_legacy":"public_legacy.yaml", + "ETC":"etc.yaml", + "dev_rotate_meta_packages":"astroconda.dev_rotate.yaml", + "test":"test.yaml"] +//---------------------------------------------------------------------------- + +node(this.label) { + + this.OSname = null + def uname = sh(script: "uname", returnStdout: true).trim() + if (uname == "Darwin") { + this.OSname = "MacOSX" + env.PATH = "${env.PATH}:/sw/bin" + this.CONDA_BLD_OUTPUT_DIR = "osx-64" + } + if (uname == "Linux") { + this.OSname = uname + this.CONDA_BLD_OUTPUT_DIR = "linux-64" + } + assert uname != null + + this.BUILD_SUBDIR = "${this.release_type}/${this.py_version}" + this.BUILD_ROOT = "${env.WORKSPACE}/${this.BUILD_SUBDIR}" + + // Delete any existing job workspace directory contents. + // The directory deleted is the one named after the jenkins pipeline job. + deleteDir() + + // Perform the following build job tasks in BUILD_SUBDIR within the + // topmost workspace directory for this node to allow for multiple + // parallel jobs using different combinations of build parameters. + dir("${this.BUILD_SUBDIR}") { + + stage('Setup') { + + println "this.label ${this.label}" + assert this.label != null + assert this.label != "label-DEFAULT" + + println "this.py_version ${this.py_version}" + assert this.py_version != null + assert this.py_version != "py_version-DEFAULT" + + println "this.release_type ${this.release_type}" + assert this.release_type != null + assert this.release_type != "release_type-DEFAULT" + + // Fetch the manifest files + git url: this.build_control_URL + + // Check for the availability of a download tool and then use it + // to get the conda installer. + def dl_cmds = ["wget --no-verbose --server-response --no-check-certificate", + "curl -OSs"] + def dl_cmd = null + def stat1 = 999 + for (cmd in dl_cmds) { + stat1 = sh(script: "which ${cmd.split()[0]}", returnStatus: true) + if( stat1 == 0 ) { + dl_cmd = cmd + break + } + } + if (stat1 != 0) { + println("Could not find a download tool. Unable to proceed.") + sh "false" + } + + def conda_installer = + this.conda_installers["${this.OSname}-${this.py_version}"] + dl_cmd = dl_cmd + " ${this.conda_base_URL}${conda_installer}" + sh dl_cmd + + // Make the log files a bit more deterministic + env.PYTHONUNBUFFERED = "true" + + // Run miniconda installer and then force to particular version + sh "bash ./${conda_installer} -b -p miniconda" + env.PATH = "${this.BUILD_ROOT}/miniconda/bin/:" + "${env.PATH}" + sh "conda install --quiet conda=${this.conda_version}" + sh "conda install --quiet --yes conda-build=${this.conda_build_version}" + + // TODO: Check for presence of build support tools, such as git, + // make, compilers, etc. and print a summary of their versions + // for auditing purposes. + + this.manifest = readYaml file: "manifests/" + + this.manifest_files["${this.release_type}"] + println("Manifest repository: ${this.manifest.repository}") + println("Manifest numpy version specification: " + + "${this.manifest.numpy_version}") + println("Manifest packages to build:") + for (pkgname in this.manifest.packages) { + println(pkgname) + } + + println("Checking for supplemental channels specification:") + if (this.manifest.channels != null) { + println("Supplemental channel(s) found, adding:") + for (channel in this.manifest.channels) { + sh "conda config --add channels ${channel}" + } + } else { + println("[INFO] No supplemental channels specified in manifest.") + } + + // Retrieve conda recipes + def recipes_dir = "conda-recipes" + dir(recipes_dir) { + git url: this.manifest.repository + } + + // Set git ID info to avoid errors with commands like 'git tag'. + sh "git config user.email \"ssb-jenkins@ssb-jenkins\"" + sh "git config user.name \"ssb-jenkins\"" + } + + stage('Build packages') { + def build_cmd = "conda build" + def pyversion_num = this.py_version.split('y')[1] + def build_args = "--no-anaconda-upload --python=${pyversion_num}" + + " --numpy=${this.manifest.numpy_version} --skip-existing" + def stat2 = 999 + dir ("conda-recipes") { + for (pkgname in this.manifest.packages) { + stat2 = sh(script: "${build_cmd} ${build_args} ${pkgname}", + returnStatus: true) + println("Shell call returned status: ${stat2}") + println "\nEnd Build: ${pkgname} " + + "=======================================================" + } + } + + // Determine if each package in the manifest was built. + // Set flag to fail the stage in the event of any missing packages. + def stage_success = "true" + def outdir = "miniconda/conda-bld/${this.CONDA_BLD_OUTPUT_DIR}" + dir(outdir) { + def built_pkgs = sh(script: "ls -1 *.tar.bz2", + returnStdout: true).trim().split() + def found_pkg = false + def first = true + for (pkgname in this.manifest.packages) { + found_pkg = false + for (built_pkg in built_pkgs) { + if ( built_pkg.indexOf(pkgname, 0) != -1) { + found_pkg = true + break + } + } + if (!found_pkg) { + if (first) { + println("ERROR BUILDING ONE OR MORE PACKAGES!\n") + first = false + } + println("ERROR building: ${pkgname}") + stage_success = "false" + } + } + } + sh stage_success + } + + stage('Tests') { + println "Test results go here." + } + + stage('Publish/Archive build products') { + // Publishing and/or archival steps go here. + } + + } // end dir() + +} //end node + diff --git a/jenkins/package_builder.groovy b/jenkins/package_builder.groovy index 92fbae4..55784a3 100644 --- a/jenkins/package_builder.groovy +++ b/jenkins/package_builder.groovy @@ -12,7 +12,6 @@ node(this.label) { " Package Build Info Summary:\n" + "${time}\n" + "JOB_DEF_GENERATION_TIME: ${JOB_DEF_GENERATION_TIME}\n" + - "inherited workspace: ${this.parent_workspace}\n" + "this.Nodelabel: ${this.label}\n" + "env.JOB_BASE_NAME: ${env.JOB_BASE_NAME}\n" + "env.JOB_NAME: ${env.JOB_NAME}\n" + @@ -20,7 +19,11 @@ node(this.label) { "env.NODE_NAME: ${env.NODE_NAME}\n" + "env.WORKSPACE: ${env.WORKSPACE}\n" + "env.JENKINS_HOME: ${env.JENKINS_HOME}\n" + + "parameter parent_workspace: ${this.parent_workspace}\n" + "parameter py_version: ${this.py_version}\n" + + "parameter numpy_version: ${this.numpy_version}\n" + + "parameter cull_manifest: ${this.cull_manifest}\n" + + "parameter channel_URL: ${this.channel_URL}\n" + "PATH: ${env.PATH}\n" + "PYTHONPATH: ${env.PYTHONPATH}\n" + "PYTHONUNBUFFERED: ${env.PYTHONUNBUFFERED}\n") @@ -33,6 +36,15 @@ node(this.label) { cmd = "conda build" stage("Build") { + // Use channel URL obtained from manifest in build command if + // manifest has been culled to allow packages being built to + // simply download dependency packages from the publication + // channel as needed, rather than build them as part of the + // package build session that requires them. + def channel_option = "${this.channel_URL}" + if (this.cull_manifest == "false") { + channel_option = "" + } build_cmd = cmd args = ["--no-test", "--no-anaconda-upload", @@ -41,6 +53,7 @@ node(this.label) { "--skip-existing", "--override-channels", "--channel defaults", + "${channel_option}", "--dirty"] for (arg in args) { build_cmd = "${build_cmd} ${arg}" -- cgit From fc46dd885d9f7307de593fdef8dd672808f5f477 Mon Sep 17 00:00:00 2001 From: Matt Rendina Date: Wed, 7 Jun 2017 08:25:56 -0400 Subject: Add parameters to package_builder jobs --- jenkins/generator_DSL.groovy | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/jenkins/generator_DSL.groovy b/jenkins/generator_DSL.groovy index de58455..c14efec 100644 --- a/jenkins/generator_DSL.groovy +++ b/jenkins/generator_DSL.groovy @@ -88,6 +88,12 @@ for(pkg in config.packages) { stringParam("manifest_file", "manifest_file-DEFAULTVALUE", "Manifest (release) file to use for the build.") + stringParam("cull_manifest", + "cull_manifest-DEFAULTVALUE", + "Was the manifest culled as part of dispatch?") + stringParam("channel_URL", + "channel_URL-DEFAULTVALUE", + "Publication channel used for culled builds.") } definition { cps { -- cgit From d83a8b50f892f99799f215a3736db9cf55bfe825 Mon Sep 17 00:00:00 2001 From: Matt Rendina Date: Wed, 7 Jun 2017 08:32:58 -0400 Subject: Add missing --channel flag --- jenkins/package_builder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins/package_builder.groovy b/jenkins/package_builder.groovy index 55784a3..37812ec 100644 --- a/jenkins/package_builder.groovy +++ b/jenkins/package_builder.groovy @@ -41,7 +41,7 @@ node(this.label) { // simply download dependency packages from the publication // channel as needed, rather than build them as part of the // package build session that requires them. - def channel_option = "${this.channel_URL}" + def channel_option = "--channel ${this.channel_URL}" if (this.cull_manifest == "false") { channel_option = "" } -- cgit