1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
|
// Parameters inherited from the calling script via environment injection.
//----------------------------------------------------------------------------
// MANIFEST_FILE - The "release" type; list of recipes/packages to build
// LABEL - Node or logical group of build nodes
// PY_VERSION - Python version hosted by conda to support the build
// BUILD_CONTROL_REPO - Repository holding this & other build system files,
// and manifest files
// BUILD_CONTROL_BRANCH - Branch to obtain from build control repo
// CONDA_VERSION - First, then the version is forced to this value.
// CONDA_BUILD_VERSION - Conda-build is installed forced to this version.
// CONDA_BASE_URL - Where to get the conda installer
// UTILS_REPO - Repository holding support utilities
// Directories to create within the workspace
this.utils_dir = "utils"
this.recipes_dir = "conda-recipes"
// The conda installer script to use for various <OS><py_version> combinations.
this.conda_installers = ["Linux-py2.7":"Miniconda2-${CONDA_VERSION}-Linux-x86_64.sh",
"Linux-py3.5":"Miniconda3-${CONDA_VERSION}-Linux-x86_64.sh",
"MacOSX-py2.7":"Miniconda2-${CONDA_VERSION}-MacOSX-x86_64.sh",
"MacOSX-py3.5":"Miniconda3-${CONDA_VERSION}-MacOSX-x86_64.sh"]
node(LABEL) {
this.OSname = null
def uname = sh(script: "uname", returnStdout: true).trim()
if (uname == "Darwin") {
this.OSname = "MacOSX"
this.OSdesc = "MacOS"
this.OSversion = sh(script: "sw_vers -productVersion", returnStdout: true).trim()
env.PATH = "${env.PATH}:/sw/bin"
this.CONDA_PLATFORM = "osx-64"
}
if (uname == "Linux") {
this.OSname = uname
this.OSdesc = sh(script: "lsb_release -i",
returnStdout: true).trim().tokenize()[2]
this.OSversion = sh(script: "lsb_release -r",
returnStdout: true).trim().tokenize()[1]
this.CONDA_PLATFORM = "linux-64"
}
assert uname != null
// Delete any existing job workspace directory contents.
// The directory deleted is the one named after the jenkins pipeline job.
deleteDir()
// Get the manifest and build control files
git branch: BUILD_CONTROL_BRANCH, url: BUILD_CONTROL_REPO
this.manifest = readYaml file: "manifests/${MANIFEST_FILE}"
if (this.manifest.channel_URL[-1..-1] == "/") {
this.manifest.channel_URL = this.manifest.channel_URL[0..-2]
}
// Allow for sharing build_list between stages below.
this.build_list = []
stage("Summary") {
def time = new Date()
def manifest_pkg_txt = ""
for (pkgname in this.manifest.packages) {
manifest_pkg_txt = "${manifest_pkg_txt}${pkgname}\n"
}
println("\n" +
" Build Info Summary:\n" +
"${time}\n" +
"JOB_DEF_GENERATION_TIME: ${JOB_DEF_GENERATION_TIME}\n" +
"OSname: ${this.OSname}\n" +
"OSDesc: ${this.OSdesc}\n" +
"OSverion: ${this.OSversion}\n" +
"script: dispatch.groovy\n" +
"env.WORKSPACE: ${env.WORKSPACE}\n" +
" Job suite parameters:\n" +
"LABEL: ${LABEL}\n" +
"env.NODE_NAME: ${env.NODE_NAME}\n" +
"PY_VERSION: ${PY_VERSION}\n" +
"MANIFEST_FILE: ${MANIFEST_FILE}\n" +
"PATH: ${PATH}\n" +
"CONDA_VERSION: ${CONDA_VERSION}\n" +
"CONDA_BUILD_VERSION: ${CONDA_BUILD_VERSION}\n" +
"CONDA_BASE_URL: ${CONDA_BASE_URL}\n" +
"BUILD_CONTROL_REPO: ${BUILD_CONTROL_REPO}\n" +
"BUILD_CONTROL_BRANCH: ${BUILD_CONTROL_BRANCH}\n" +
"UTILS_REPO: ${UTILS_REPO}\n" +
" Trigger parameters:\n" +
"this.cull_manifest: ${this.cull_manifest}\n" +
" Manifest values:\n" +
"Recipe repository: ${this.manifest.repository}\n" +
"Numpy version spec: ${this.manifest.numpy_version}\n" +
"Channel URL: ${this.manifest.channel_URL}\n" +
"Package list:\n${manifest_pkg_txt}")
}
stage("Setup") {
// Inherited from env() assignment performed in the generator
// DSL script.
assert LABEL != null
assert LABEL != "label-DEFAULTVALUE"
// Inherited from env() assignment performed in the generator
// DSL script.
assert PY_VERSION != null
assert PY_VERSION != "py_version-DEFAULTVALUE"
this.py_maj_version = "${PY_VERSION.tokenize(".")[0]}"
// Inherited from env() assignment performed in the generator
// DSL script.
assert MANIFEST_FILE != null
assert MANIFEST_FILE != "manifest_file-DEFAULTVALUE"
// Get conda recipes
dir(this.recipes_dir) {
git url: this.manifest.repository
}
// Get build utilities
dir(this.utils_dir) {
git url: UTILS_REPO
sh "python setup.py install"
}
// Check for the availability of a download tool and then use it
// to get the conda installer.
if (CONDA_BASE_URL[-1..-1] == "/") {
CONDA_BASE_URL = [0..-2]
}
def dl_cmds = ["wget --no-verbose --server-response --no-check-certificate",
"curl -OSs"]
def dl_cmd = null
def stat1 = 999
for (cmd in dl_cmds) {
stat1 = sh(script: "which ${cmd.tokenize()[0]}", returnStatus: true)
if( stat1 == 0 ) {
dl_cmd = cmd
break
}
}
if (stat1 != 0) {
println("Could not find a download tool. Unable to proceed.")
sh "false"
}
def conda_installer =
this.conda_installers["${this.OSname}-py${PY_VERSION}"]
dl_cmd = dl_cmd + " ${CONDA_BASE_URL}/${conda_installer}"
sh dl_cmd
// Install specific versions of miniconda and conda-build
sh "bash ./${conda_installer} -b -p miniconda"
env.PATH = "${env.WORKSPACE}/miniconda/bin/:${env.PATH}"
sh "conda install --quiet conda=${CONDA_VERSION}"
sh "conda install --quiet --yes conda-build=${CONDA_BUILD_VERSION}"
// Apply bugfix patch to conda_build 2.1.1
def filename = "${env.WORKSPACE}/miniconda/lib/python${PY_VERSION}/" +
"site-packages/conda_build/config.py"
def patches_dir = "${env.WORKSPACE}/patches"
def patchname = "conda_build_2.1.1_substr_fix_py${this.py_maj_version}.patch"
def full_patchname = "${patches_dir}/${patchname}"
sh "patch ${filename} ${full_patchname}"
}
stage("Generate build list") {
// Generate a filtered, optionally culled, & dependency-ordered list
// of available package recipes.
def culled_option = "--culled"
if (this.cull_manifest == "false") {
culled_option = ""
}
def build_list_file = "build_list"
cmd = "rambo"
args = ["--platform ${this.CONDA_PLATFORM}",
"--python ${PY_VERSION}",
"--manifest manifests/${MANIFEST_FILE}",
"--file ${build_list_file}",
"${culled_option}",
this.recipes_dir]
for (arg in args) {
cmd = "${cmd} ${arg}"
}
sh(script: cmd)
def build_list_text = readFile build_list_file
this.build_list = build_list_text.trim().tokenize()
println("Build list:")
println(build_list_text)
}
stage("Build packages") {
for (pkg in this.build_list) {
build job: pkg,
parameters:
[string(name: "label", value: env.NODE_NAME),
string(name: "py_version", value: PY_VERSION),
string(name: "numpy_version",
value: "${this.manifest.numpy_version}"),
string(name: "parent_workspace", value: env.WORKSPACE)],
propagate: false
}
}
}
|
