diff options
Diffstat (limited to 'jwstdp/utils')
| -rwxr-xr-x | jwstdp/utils/archive_cal_code.sh | 71 | ||||
| -rwxr-xr-x | jwstdp/utils/credential_helper.sh | 3 | ||||
| -rwxr-xr-x | jwstdp/utils/jwst_delivery_prep | 205 | ||||
| -rw-r--r-- | jwstdp/utils/params.cfg | 4 |
4 files changed, 283 insertions, 0 deletions
diff --git a/jwstdp/utils/archive_cal_code.sh b/jwstdp/utils/archive_cal_code.sh new file mode 100755 index 0000000..d41372b --- /dev/null +++ b/jwstdp/utils/archive_cal_code.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash + +# A tool for generating a JWST CAL software tarball and the associated +# project code linecount summary and depositing it into the provided +# archival area. + +if [[ $# -eq 0 ]]; +then + printf "\nUsage:\n" + printf "$(basename $0) <jwst-tag> <destination-base-path>\n\n" + exit 0 +fi + +if [[ $1 == '' ]]; +then + echo "JWST repository tag is required" + exit 1 +fi +tag=$1 + +if [[ $2 == '' ]]; +then + echo "Destination root directory is required" + exit 1 +fi +dest_root=$2 # dms_x.y(.z) + +code_dest_path=${dest_root}/software_bundles +doc_dest_path=${dest_root}/code_counts +if [[ ! -d $code_dest_path ]]; +then + echo "${code_dest_path} does not exist!" + exit 1 +fi +if [[ ! -d $doc_dest_path ]]; +then + echo "${doc_dest_path} does not exist!" + exit 1 +fi + +which sloccount +if [[ $? -ne 0 ]]; +then + echo "'sloccount' not found. Install before attempting again." + exit 1 +fi +which csplit +if [[ $? -ne 0 ]]; +then + echo "'csplit' not found. Install before attempting again." + exit 1 +fi + +set -e + +local_checkout_dir=jwst-cal-archive +git clone https://github.com/stsci-jwst/jwst.git $local_checkout_dir +pushd $local_checkout_dir +git checkout $tag + +# Create the line count summary. Strip off any superfluous leading text to +# leave only the report data itself. +countname="jwst-${tag}-sloccount.txt" +sloccount . > temp.txt 2>&1 +csplit --digits=0 --prefix $countname temp.txt %SLOC% +mv ${countname}0 ${countname} + +python setup.py sdist + +cp -a dist/jwst-${tag}.tar.gz ${code_dest_path} +cp -a $countname ${doc_dest_path} diff --git a/jwstdp/utils/credential_helper.sh b/jwstdp/utils/credential_helper.sh new file mode 100755 index 0000000..1e69279 --- /dev/null +++ b/jwstdp/utils/credential_helper.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +echo username=$USERNAME +echo password=$PASSWORD diff --git a/jwstdp/utils/jwst_delivery_prep b/jwstdp/utils/jwst_delivery_prep new file mode 100755 index 0000000..8183eff --- /dev/null +++ b/jwstdp/utils/jwst_delivery_prep @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 + +# Collects environment snapshot files from the internal artifactory instance +# at https://bytesalad.stsci.edu for a successful JWST regression test (RT) +# run, and stores them in a new release directory in a clone of the releases +# repository along with a descriptive README file as part of a JWSTDP +# environment delivery process. + +import os +import sys +import argparse +import subprocess as sp +from urllib import request +import re +import configparser + + +def modify_dep(line, text, jwstpip=True): + '''Replace value appearing after the last '@' in the pip + spec line for the 'jwst' package with the provided text.''' + line = line.strip() + # Determine type of pip dependency spec + if line[0:2] == '-e': + print('editable install dependency spec') + if line[0:4] == 'jwst': + if jwstpip: + line = f'jwst=={text}' + else: + delim = line.rfind('@') + line = line[:delim+1] + line = line + text + return(line) + + +def get_artifact_names(url, prefixes): + '''Retrieve list of all available artifacts in the target artifactory + repository.''' + names = [] + req = request.Request(url) + result = request.urlopen(req) + payload = result.readlines() + for line in payload: + line = str(line.decode()) + for prefix in prefixes: + if prefix in line: + mat = re.search('(?<=").*(?=\")', line) + names.append(mat.group(0)) + return(names) + + +def write_readme(release_tag, config_map, filename): + ''' Write a descriptive README.md file customized for this release.''' + with open(filename, 'w') as f: + f.write(("# Installing the tested pipeline stack\n" + "\n" + "Conda (miniconda3 or anaconda3) must already be installed, if it is not,\n" + "'Advance Setup' below.\n" + "All steps must be performed in bash or a compatible shell.\n" + "\n" + "A fresh installation of Anaconda3 or Miniconda3 is not required for each JWSTDP\n" + "release. The method described here allows for multiple, entirely segregated,\n" + "pipeline installations.\n" + "\n")) + + for cset in config_map: + procedure = (f"## {cset.os}\n" + f"To reproduce the environment used during JWST prerelease testing on Linux, a \n" + f"three-step installation process is required.\n" + f"\n" + f"1) Install the target python interpreter and its dependencies using conda, then\n" + f"```\n" + f"$ conda create -n jwstdp-{release_tag} --file\n" + f"https://ssb.stsci.edu/releases/jwstdp/{release_tag}/conda_python_{cset.config}.txt\n" + f"```\n" + f"\n" + f"2) Activate the environment\n" + f"```\n" + f"$ source activate jwstdp-{release_tag}\n" + f"```\n" + f"\n" + f"3) Install the pipeline software packages on top using `pip`:\n" + f"```\n" + f"$ pip install -r https://ssb.stsci.edu/releases/jwstdp/{release_tag}/reqs_{cset.config}.txt\n" + f"```\n" + f"\n") + f.write(procedure) + + f.write("# Advance setup\n" + " \n" + "If conda has not yet been installed, use the following steps to obtain\n" + "it, then use the procedure above to install the pipeline software.\n" + "\n" + "For detailed instructions of this step, please visit: http://docs.continuum.io/anaconda/install#linux-install\n" + "\n" + "**For Miniconda:**\n" + "\n" + "```\n" + "$ wget\n" + "https://repo.continuum.io/miniconda/Miniconda3-Latest-Linux-x86_64.sh\n" + "$ bash Miniconda3-Latest-Linux-x86_64.sh\n" + "$ export PATH=$HOME/miniconda3/bin:$PATH\n" + "```\n" + "\n" + "**For Anaconda (if preferred):**\n" + "\n" + "```\n" + "$ wget\n" + "https://repo.continuum.io/archive/Anaconda3-2019.10-Linux-x86_64.sh\n" + "$ bash Anaconda3-2019.10-Linux-x86_64.sh\n" + "$ export PATH=$HOME/anaconda3/bin:$PATH\n" + "```\n") + + +def main(): + ap = argparse.ArgumentParser( + prog='jwst_pipeline_deliver', + description='Modify regression test environment snapshot artifacts' + ' to reflect a stable release tag for all supported OSs, compose ' + 'a descriptive readme file, and store delivery files in a new ' + 'release directory within the repository holding this script.') + ap.add_argument('--tag', + '-t', + type=str, + required=True, + help='Tag used for the target release of the JWST package.') + ap.add_argument('--config', + '-c', + required=False, + help='Parameters file for configuring the behavior of this' + ' script. If this flag is not used, the default config ' + 'file name "params.cfg" in the same directory as the ' + 'script will be read.') + ap.add_argument('build_names', + help='BuildConfig names for which artifacts will be ' + 'collected', + nargs='+') + args = ap.parse_args() + + if len(args.build_names) == 0: + print('One or more build names are required as argument.') + sys.exit(1) + + scriptdir = sys.path[0] + + configfile = args.config + if configfile is None: + configfile = 'params.cfg' + + config = configparser.ConfigParser() + config.read(os.path.join(scriptdir, configfile)) + artifact_prefixes = config['main']['artifact_prefixes'].split() + art_url_base = config['main']['art_url_base'] + art_repo = config['main']['art_repo'] + + release_tag = args.tag + + # Crete new release directory + reldir = f'{scriptdir}/../{release_tag}' + os.mkdir(reldir) + + startdir = os.getcwd() + os.chdir(reldir) + + config_map = [] + from collections import namedtuple + confset = namedtuple('confset', ['config', 'os']) + artifacts = get_artifact_names( + f'{art_url_base}/{art_repo}', + artifact_prefixes) + + # Download only the available artifacts that correspond to the + # requested build name into new release dir. + for config in args.build_names: + for artifact in artifacts: + for prefix in artifact_prefixes: + if artifact == prefix+config+'.txt': + aurl = f'{art_url_base}/{art_repo}/{artifact}' + req = request.Request(aurl) + result = request.urlopen(req) + payload = result.readlines() + + # Determine OS of artifact and map to config name. + if 'linux-64' in str(payload[-1]): + cset = confset(config=config, os='Linux') + config_map.append(cset) + if 'osx-64' in str(payload[-1]): + cset = confset(config=config, os='Macos') + config_map.append(cset) + + # Replace jwst URL git hash with release tag name + with open(artifact, 'w') as f: + for line in payload: + line = str(line.decode()) + line = modify_dep(line, release_tag) + f.write(f'{line}\n') + + write_readme(release_tag, config_map, 'README.md') + + # Output the build_config to OS mappings. + for cset in config_map: + print(f"{cset.config}:{cset.os}") + +if __name__ == '__main__': + main() + diff --git a/jwstdp/utils/params.cfg b/jwstdp/utils/params.cfg new file mode 100644 index 0000000..3cca334 --- /dev/null +++ b/jwstdp/utils/params.cfg @@ -0,0 +1,4 @@ +[main] +artifact_prefixes = conda_python_ reqs_ +art_url_base = https://bytesalad.stsci.edu/artifactory +art_repo = jwst-pipeline-results |