diff options
Diffstat (limited to 'jenkins')
| -rw-r--r-- | jenkins/dispatch.groovy | 19 | ||||
| -rw-r--r-- | jenkins/generator_DSL.groovy | 50 | ||||
| -rw-r--r-- | jenkins/multi_suite_generator.groovy | 81 | ||||
| -rw-r--r-- | jenkins/package_builder.groovy | 3 |
4 files changed, 107 insertions, 46 deletions
diff --git a/jenkins/dispatch.groovy b/jenkins/dispatch.groovy index 49207c2..89aa26f 100644 --- a/jenkins/dispatch.groovy +++ b/jenkins/dispatch.groovy @@ -7,8 +7,7 @@ // NUMPY_VERSION - numpy version used to support the build // BUILD_CONTROL_REPO - Repository holding this & other build system files, // and manifest files -// BUILD_CONTROL_BRANCH - Branch to obtain from build control repo -// BUILD_CONTROL_TAG - Tag to obtain from build control repo +// BUILD_CONTROL_GIT_REF - Git ref to use in obtaining build control scripts // CONDA_INSTALLER_VERSION - Conda installer version to use // CONDA_VERSION - conda version is forced to this value // CONDA_BUILD_VERSION - Conda-build is installed forced to this version. @@ -87,13 +86,11 @@ node(LABEL) { sh "env | sort" // Get the build control files - git branch: BUILD_CONTROL_BRANCH, url: BUILD_CONTROL_REPO + git url: BUILD_CONTROL_REPO - // If a tag was specified in the job-suite-generator configuration, - // explicitly check out that tag after cloning the (master) branch, - // since the 'git' pipeline step does not yet support accessing tags. - if (BUILD_CONTROL_TAG != "") { - sh(script: "git checkout tags/${BUILD_CONTROL_TAG}") + // The 'git' pipeline step does not yet support accessing tags. + if (BUILD_CONTROL_GIT_REF != "") { + sh(script: "git checkout ${BUILD_CONTROL_GIT_REF}") } // Turn multi-line env var delimiters into simple newlines for @@ -141,8 +138,7 @@ node(LABEL) { "CONDA_BUILD_VERSION: ${CONDA_BUILD_VERSION}\n" + "CONDA_BASE_URL: ${CONDA_BASE_URL}\n" + "BUILD_CONTROL_REPO: ${BUILD_CONTROL_REPO}\n" + - "BUILD_CONTROL_BRANCH: ${BUILD_CONTROL_BRANCH}\n" + - "BUILD_CONTROL_TAG: ${BUILD_CONTROL_TAG}\n" + + "BUILD_CONTROL_GIT_REF: ${BUILD_CONTROL_GIT_REF}\n" + "UTILS_REPO: ${UTILS_REPO}\n" + " Trigger parameters:\n" + "this.cull_manifest: ${this.cull_manifest}\n" + @@ -308,8 +304,7 @@ node(LABEL) { parameters: [ string(name: "label", value: env.NODE_NAME), string(name: "build_control_repo", value: BUILD_CONTROL_REPO), - string(name: "build_control_branch", value: BUILD_CONTROL_BRANCH), - string(name: "build_control_tag", value: BUILD_CONTROL_TAG), + string(name: "build_control_git_ref", value: BUILD_CONTROL_GIT_REF), string(name: "py_version", value: PY_VERSION), string(name: "numpy_version", value: NUMPY_VERSION), string(name: "parent_workspace", value: env.WORKSPACE), diff --git a/jenkins/generator_DSL.groovy b/jenkins/generator_DSL.groovy index c18085c..0ad86ac 100644 --- a/jenkins/generator_DSL.groovy +++ b/jenkins/generator_DSL.groovy @@ -1,5 +1,7 @@ -// Job generator script. Uses Job-DSL plugin API. // Third party YAML parsing class. Obtain from URL below before use. -// https://repo1.maven.org/maven2/org/yaml/snakeyaml/1.17/snakeyaml-1.17.jar +// TODO: Is there a way to reference this local, shaded class instead of fetching it from an external source +// using @Grab? +//import org.jenkinsci.plugins.pipeline.utility.steps.shaded.org.yaml.snakeyaml.Yaml // cannot find. +@Grab('org.yaml:snakeyaml:1.17') import org.yaml.snakeyaml.Yaml def yaml = new Yaml() @@ -18,23 +20,13 @@ def job_def_generation_time = new Date() this.script = "dispatch.groovy" -this.build_control_repo = readFileFromWorkspace("VAR-build_control_repo") -this.build_control_repo = this.build_control_repo.trim() - -this.build_control_branch = readFileFromWorkspace("VAR-build_control_branch") -this.build_control_branch= this.build_control_branch.trim() - -this.build_control_tag = readFileFromWorkspace("VAR-build_control_tag") -this.build_control_tag = this.build_control_tag.trim() - // Keep a specified number of builds (purging those older upon next // job execution) for each independent job that is created. This value is set // as one of the updater job's parameters. -// Note: Since the _dispatch job is -// executed at the highest frequency, its saved build logs will not go as far -// back in time for a given value of builds_to_keep than a similar collection -// of kept logs for a less frequently built job, a slowly-moving package, for -// instance. +// Note: Since the _dispatch job is executed at the highest frequency, +// its saved build logs will not go as far back in time for a given value of +// builds_to_keep than a similar collection of kept logs for a less frequently +// built job, a slowly-moving package, for instance. println("builds_to_keep: ${builds_to_keep}") this.num_builds_to_keep = builds_to_keep.toInteger() @@ -42,18 +34,17 @@ this.num_builds_to_keep = builds_to_keep.toInteger() // over each python version provided by the 'py_versions' job parameter, to obtain // every combination of OS and python version. Generate a separate job suite for // each combination. -for (label in labels.trim().tokenize()) { - for (py_version in py_versions.trim().tokenize()) { - for (numpy_version in numpy_versions.trim().tokenize()) { - +for (label in labels) { + for (py_version in py_versions) { + for (numpy_version in numpy_versions) { + //----------------------------------------------------------------------- // Create a folder to contain the jobs which are created below. suite_name = "${manifest_file.tokenize(".")[0]}_${label}_py${py_version}_np${numpy_version}" folder(suite_name) { description("Build suite generated: ${job_def_generation_time}\n" + "build control repo: ${build_control_repo}\n" + - "build control branch: ${build_control_branch}\n" + - "build control tag: ${build_control_tag}\n" + + "build control git_ref: ${build_control_git_ref}\n" + "conda installer version: ${conda_installer_version}\n" + "conda version: ${conda_version}\n" + "conda-build version: ${conda_build_version}\n" + @@ -87,8 +78,7 @@ for (label in labels.trim().tokenize()) { "PY_VERSION: ${py_version}\n" + "NUMPY_VERSION: ${numpy_version}\n" + "BUILD_CONTROL_REPO: ${build_control_repo}\n" + - "BUILD_CONTROL_BRANCH: ${build_control_branch}\n" + - "BUILD_CONTROL_TAG: ${build_control_tag}\n" + + "BUILD_CONTROL_GIT_REF: ${build_control_git_ref}\n" + "CONDA_INSTALLER_VERSION: ${conda_installer_version}\n" + "CONDA_VERSION: ${conda_version}\n" + "CONDA_BUILD_VERSION: ${conda_build_version}\n" + @@ -103,8 +93,7 @@ for (label in labels.trim().tokenize()) { env("PY_VERSION", py_version) env("NUMPY_VERSION", numpy_version) env("BUILD_CONTROL_REPO", build_control_repo) - env("BUILD_CONTROL_BRANCH", build_control_branch) - env("BUILD_CONTROL_TAG", build_control_tag) + env("BUILD_CONTROL_GIT_REF", build_control_git_ref) env("CONDA_INSTALLER_VERSION", conda_installer_version) env("CONDA_VERSION", conda_version) env("CONDA_BUILD_VERSION", conda_build_version) @@ -139,12 +128,9 @@ for (label in labels.trim().tokenize()) { stringParam("build_control_repo", "build_control_repo-DEFAULTVALUE", "Repository containing the build system scripts.") - stringParam("build_control_branch", - "build_control_branch-DEFAULTVALUE", - "Branch checked out to obtain build system scripts.") - stringParam("build_control_tag", - "build_control_tag-DEFAULTVALUE", - "Tag checked out to obtain build system scripts.") + stringParam("build_control_git_ref", + "build_control_git_ref-DEFAULTVALUE", + "Git ref to use to obtain the build system scripts.") stringParam("py_version", "py_version-DEFAULTVALUE", "python version to use") diff --git a/jenkins/multi_suite_generator.groovy b/jenkins/multi_suite_generator.groovy new file mode 100644 index 0000000..c2b3ef0 --- /dev/null +++ b/jenkins/multi_suite_generator.groovy @@ -0,0 +1,81 @@ +import org.jenkinsci.plugins.pipeline.utility.steps.shaded.org.yaml.snakeyaml.Yaml +// Top-level pipeline job that provides parameterized machinery for +// creating one or more build job suites for use in building AstroConda +// package sets. +// Uses Job-DSL plugin. + +// DSL script path within the repository obtained for this job. +this.dsl_script = "jenkins/generator_DSL.groovy" + +// To sidestep not-serializable exceptions, encapsulate the YAML reading +// step in a method of its own, used below. +@NonCPS +def readYaml(data) { + def yaml = new Yaml() + payload = yaml.load(data) + return payload +} + +def updates = '' + +node("master") { + + stage("Prep") { + + // Delete any existing job workspace directory contents. + deleteDir() + unstash 'build_control' + sh "env" + + // Read update/generate YAML description. + updates = readYaml(update_list) + + // Clone the manifests repository. + dir ("manifests") { + git url: manifest_repo + } + + build_control_git_ref = git_ref.trim() + + // 'Parameters' variables are provided by the execution of the + // generator build task with parameters. Each is populated by a + // parameter specification at job execution time. Varaiables defined as + // build parameters for this job are automatically available in the + // called JobDSL script invoked below by using their base name, i.e. + // the name here without a 'this.' prefix. Other variables are not + // automatically available, see above. + println(" From job config:\n" + + "build_control_repo: ${build_control_repo}\n" + + "build_control_git_ref: ${build_control_git_ref}\n" + + " Parameters:\n" + + "conda_installer_version: ${this.conda_installer_version}\n" + + "conda_version: ${this.conda_version}\n" + + "conda_build_version: ${this.conda_build_version}\n" + + "conda_base_URL: ${this.conda_base_URL}\n" + + "utils_repo: ${this.utils_repo}\n" + + "old_jobs_action: ${this.old_jobs_action}\n" + + " Other values:\n" + + "dsl_script: ${this.dsl_script}") + } + + // Loop over requested job suites to generate or update. + stage("Spawn job definitions") { + def add_params = [:] + add_params.put('build_control_repo', build_control_repo) + add_params.put('build_control_git_ref', build_control_git_ref) + + updates.keySet().each { key, val -> + manifest_file = "${key}.yaml" + add_params.put('manifest_file', manifest_file) + add_params.put('labels', updates[key].labels) + add_params.put('py_versions', updates[key].py_versions) + add_params.put('numpy_versions', updates[key].numpy_versions) + println("Generating job suite for: ${this.manifest_file}") + jobDsl targets: [this.dsl_script].join("\n"), + lookupStrategy: "SEED_JOB", + additionalParameters: add_params, + removeAction: "${this.old_jobs_action}" + } + } + +} diff --git a/jenkins/package_builder.groovy b/jenkins/package_builder.groovy index fb7e251..39bb9e0 100644 --- a/jenkins/package_builder.groovy +++ b/jenkins/package_builder.groovy @@ -39,8 +39,7 @@ node(this.label) { "env.WORKSPACE: ${env.WORKSPACE}\n" + "env.JENKINS_HOME: ${env.JENKINS_HOME}\n" + "parameter build_control_repo: ${this.build_control_repo}\n" + - "parameter build_control_branch: ${this.build_control_branch}\n" + - "parameter build_control_tag: ${this.build_control_tag}\n" + + "parameter build_control_git_ref: ${this.build_control_git_ref}\n" + "parameter parent_workspace: ${this.parent_workspace}\n" + "parameter py_version: ${this.py_version}\n" + "parameter numpy_version: ${this.numpy_version}\n" + |