1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
|
#!/usr/bin/env python3
# Collects environment snapshot files from the internal artifactory instance
# at https://bytesalad.stsci.edu for a successful JWST regression test (RT)
# run, and stores them in a new release directory in a clone of the releases
# repository along with a descriptive README file as part of a JWSTDP
# environment delivery process.
import os
import sys
import argparse
import subprocess as sp
from urllib import request
import re
import configparser
def modify_dep(line, text, jwstpip=True):
'''Replace value appearing after the last '@' in the pip
spec line for the 'jwst' package with the provided text.'''
line = line.strip()
# Determine type of pip dependency spec
if line[0:2] == '-e':
print('editable install dependency spec')
if line[0:4] == 'jwst':
if jwstpip:
line = f'jwst=={text}'
else:
delim = line.rfind('@')
line = line[:delim+1]
line = line + text
return(line)
def get_artifact_names(url, prefixes):
'''Retrieve list of all available artifacts in the target artifactory
repository.'''
names = []
req = request.Request(url)
result = request.urlopen(req)
payload = result.readlines()
for line in payload:
line = str(line.decode())
for prefix in prefixes:
if prefix in line:
mat = re.search('(?<=").*(?=\")', line)
names.append(mat.group(0))
return(names)
def write_readme(release_tag, config_map, filename):
''' Write a descriptive README.md file customized for this release.'''
with open(filename, 'w') as f:
f.write(("# Installing the tested pipeline stack\n"
"\n"
"Conda (miniconda3 or anaconda3) must already be installed, if it is not,\n"
"'Advance Setup' below.\n"
"All steps must be performed in bash or a compatible shell.\n"
"\n"
"A fresh installation of Anaconda3 or Miniconda3 is not required for each JWSTDP\n"
"release. The method described here allows for multiple, entirely segregated,\n"
"pipeline installations.\n"
"\n"))
for cset in config_map:
procedure = (f"## {cset.os}\n"
f"To reproduce the environment used during JWST prerelease testing on Linux, a \n"
f"three-step installation process is required.\n"
f"\n"
f"1) Install the target python interpreter and its dependencies using conda, then\n"
f"```\n"
f"$ conda create -n jwstdp-{release_tag} --file\n"
f"https://ssb.stsci.edu/releases/jwstdp/{release_tag}/conda_python_{cset.config}.txt\n"
f"```\n"
f"\n"
f"2) Activate the environment\n"
f"```\n"
f"$ source activate jwstdp-{release_tag}\n"
f"```\n"
f"\n"
f"3) Install the pipeline software packages on top using `pip`:\n"
f"```\n"
f"$ pip install -r https://ssb.stsci.edu/releases/jwstdp/{release_tag}/reqs_{cset.config}.txt\n"
f"```\n"
f"\n")
f.write(procedure)
f.write("# Advance setup\n"
" \n"
"If conda has not yet been installed, use the following steps to obtain\n"
"it, then use the procedure above to install the pipeline software.\n"
"\n"
"For detailed instructions of this step, please visit: http://docs.continuum.io/anaconda/install#linux-install\n"
"\n"
"**For Miniconda:**\n"
"\n"
"```\n"
"$ wget\n"
"https://repo.continuum.io/miniconda/Miniconda3-Latest-Linux-x86_64.sh\n"
"$ bash Miniconda3-Latest-Linux-x86_64.sh\n"
"$ export PATH=$HOME/miniconda3/bin:$PATH\n"
"```\n"
"\n"
"**For Anaconda (if preferred):**\n"
"\n"
"```\n"
"$ wget\n"
"https://repo.continuum.io/archive/Anaconda3-2019.10-Linux-x86_64.sh\n"
"$ bash Anaconda3-2019.10-Linux-x86_64.sh\n"
"$ export PATH=$HOME/anaconda3/bin:$PATH\n"
"```\n")
def main():
ap = argparse.ArgumentParser(
prog='jwst_pipeline_deliver',
description='Modify regression test environment snapshot artifacts'
' to reflect a stable release tag for all supported OSs, compose '
'a descriptive readme file, and store delivery files in a new '
'release directory within the repository holding this script.')
ap.add_argument('--tag',
'-t',
type=str,
required=True,
help='Tag used for the target release of the JWST package.')
ap.add_argument('--config',
'-c',
required=False,
help='Parameters file for configuring the behavior of this'
' script. If this flag is not used, the default config '
'file name "params.cfg" in the same directory as the '
'script will be read.')
ap.add_argument('build_names',
help='BuildConfig names for which artifacts will be '
'collected',
nargs='+')
args = ap.parse_args()
if len(args.build_names) == 0:
print('One or more build names are required as argument.')
sys.exit(1)
scriptdir = sys.path[0]
configfile = args.config
if configfile is None:
configfile = 'params.cfg'
config = configparser.ConfigParser()
config.read(os.path.join(scriptdir, configfile))
artifact_prefixes = config['main']['artifact_prefixes'].split()
art_url_base = config['main']['art_url_base']
art_repo = config['main']['art_repo']
release_tag = args.tag
# Crete new release directory
reldir = f'{scriptdir}/../{release_tag}'
os.mkdir(reldir)
startdir = os.getcwd()
os.chdir(reldir)
config_map = []
from collections import namedtuple
confset = namedtuple('confset', ['config', 'os'])
artifacts = get_artifact_names(
f'{art_url_base}/{art_repo}',
artifact_prefixes)
# Download only the available artifacts that correspond to the
# requested build name into new release dir.
for config in args.build_names:
for artifact in artifacts:
for prefix in artifact_prefixes:
if artifact == prefix+config+'.txt':
aurl = f'{art_url_base}/{art_repo}/{artifact}'
req = request.Request(aurl)
result = request.urlopen(req)
payload = result.readlines()
# Determine OS of artifact and map to config name.
if 'linux-64' in str(payload[-1]):
cset = confset(config=config, os='Linux')
config_map.append(cset)
if 'osx-64' in str(payload[-1]):
cset = confset(config=config, os='Macos')
config_map.append(cset)
# Replace jwst URL git hash with release tag name
with open(artifact, 'w') as f:
for line in payload:
line = str(line.decode())
line = modify_dep(line, release_tag)
f.write(f'{line}\n')
write_readme(release_tag, config_map, 'README.md')
# Output the build_config to OS mappings.
for cset in config_map:
print(f"{cset.config}:{cset.os}")
if __name__ == '__main__':
main()
|
