initial commitinit

1 files changed, 205 insertions, 0 deletions

diff --git a/jwstdp/utils/jwst_delivery_prep b/jwstdp/utils/jwst_delivery_prep
new file mode 100755
index 0000000..8183eff
--- /dev/null
+++ b/jwstdp/utils/jwst_delivery_prep
@@ -0,0 +1,205 @@
+#!/usr/bin/env python3
+
+# Collects environment snapshot files from the internal artifactory instance
+# at https://bytesalad.stsci.edu for a successful JWST regression test (RT)
+# run, and stores them in a new release directory in a clone of the releases
+# repository along with a descriptive README file as part of a JWSTDP
+# environment delivery process.
+
+import os
+import sys
+import argparse
+import subprocess as sp
+from urllib import request
+import re
+import configparser
+
+
+def modify_dep(line, text, jwstpip=True):
+    '''Replace value appearing after the last '@' in the pip
+    spec line for the 'jwst' package with the provided text.'''
+    line = line.strip()
+    # Determine type of pip dependency spec
+    if line[0:2] == '-e':
+        print('editable install dependency spec')
+    if line[0:4] == 'jwst':
+        if jwstpip:
+            line = f'jwst=={text}'
+        else:
+            delim = line.rfind('@')
+            line = line[:delim+1]
+            line = line + text
+    return(line)
+
+
+def get_artifact_names(url, prefixes):
+    '''Retrieve list of all available artifacts in the target artifactory
+    repository.'''
+    names = []
+    req = request.Request(url)
+    result = request.urlopen(req)
+    payload = result.readlines()
+    for line in payload:
+        line = str(line.decode())
+        for prefix in prefixes:
+            if prefix in line:
+                mat = re.search('(?<=").*(?=\")', line)
+                names.append(mat.group(0))
+    return(names)
+
+
+def write_readme(release_tag, config_map, filename):
+    ''' Write a descriptive README.md file customized for this release.'''
+    with open(filename, 'w') as f:
+        f.write(("# Installing the tested pipeline stack\n"
+        "\n"
+        "Conda (miniconda3 or anaconda3) must already be installed, if it is not,\n"
+        "'Advance Setup' below.\n"
+        "All steps must be performed in bash or a compatible shell.\n"
+        "\n" 
+        "A fresh installation of Anaconda3 or Miniconda3 is not required for each JWSTDP\n"
+        "release. The method described here allows for multiple, entirely segregated,\n"
+        "pipeline installations.\n"
+        "\n"))
+
+        for cset in config_map:
+            procedure = (f"## {cset.os}\n"
+            f"To reproduce the environment used during JWST prerelease testing on Linux, a \n"
+            f"three-step installation process is required.\n"
+            f"\n"
+            f"1) Install the target python interpreter and its dependencies using conda, then\n"
+            f"```\n"
+            f"$ conda create -n jwstdp-{release_tag} --file\n"
+            f"https://ssb.stsci.edu/releases/jwstdp/{release_tag}/conda_python_{cset.config}.txt\n"
+            f"```\n"
+            f"\n"
+            f"2) Activate the environment\n"
+            f"```\n"
+            f"$ source activate jwstdp-{release_tag}\n"
+            f"```\n"
+            f"\n"
+            f"3) Install the pipeline software packages on top using `pip`:\n"
+            f"```\n"
+            f"$ pip install -r https://ssb.stsci.edu/releases/jwstdp/{release_tag}/reqs_{cset.config}.txt\n"
+            f"```\n"
+            f"\n")
+            f.write(procedure)
+
+        f.write("# Advance setup\n"
+        " \n"
+        "If conda has not yet been installed, use the following steps to obtain\n"
+        "it, then use the procedure above to install the pipeline software.\n"
+        "\n"
+        "For detailed instructions of this step, please visit: http://docs.continuum.io/anaconda/install#linux-install\n"
+        "\n"
+        "**For Miniconda:**\n"
+        "\n"
+        "```\n"
+        "$ wget\n"
+        "https://repo.continuum.io/miniconda/Miniconda3-Latest-Linux-x86_64.sh\n"
+        "$ bash Miniconda3-Latest-Linux-x86_64.sh\n"
+        "$ export PATH=$HOME/miniconda3/bin:$PATH\n"
+        "```\n"
+        "\n"
+        "**For Anaconda (if preferred):**\n"
+        "\n"
+        "```\n"
+        "$ wget\n"
+        "https://repo.continuum.io/archive/Anaconda3-2019.10-Linux-x86_64.sh\n"
+        "$ bash Anaconda3-2019.10-Linux-x86_64.sh\n"
+        "$ export PATH=$HOME/anaconda3/bin:$PATH\n"
+        "```\n")
+
+
+def main():
+    ap = argparse.ArgumentParser(
+            prog='jwst_pipeline_deliver',
+            description='Modify regression test environment snapshot artifacts'
+            ' to reflect a stable release tag for all supported OSs, compose '
+            'a descriptive readme file, and store delivery files in a new '
+            'release directory within the repository holding this script.')
+    ap.add_argument('--tag',
+                    '-t',
+                    type=str,
+                    required=True,
+                    help='Tag used for the target release of the JWST package.')
+    ap.add_argument('--config',
+                    '-c',
+                    required=False,
+                    help='Parameters file for configuring the behavior of this'
+                    ' script. If this flag is not used, the default config '
+                    'file name "params.cfg" in the same directory as the '
+                    'script will be read.')
+    ap.add_argument('build_names',
+                    help='BuildConfig names for which artifacts will be '
+                    'collected',
+                    nargs='+')
+    args = ap.parse_args()
+
+    if len(args.build_names) == 0:
+        print('One or more build names are required as argument.')
+        sys.exit(1)
+
+    scriptdir = sys.path[0]
+
+    configfile = args.config
+    if configfile is None:
+        configfile = 'params.cfg'
+
+    config = configparser.ConfigParser()
+    config.read(os.path.join(scriptdir, configfile))
+    artifact_prefixes = config['main']['artifact_prefixes'].split()
+    art_url_base = config['main']['art_url_base']
+    art_repo = config['main']['art_repo']
+
+    release_tag = args.tag
+    
+    # Crete new release directory
+    reldir = f'{scriptdir}/../{release_tag}'
+    os.mkdir(reldir)
+        
+    startdir = os.getcwd()
+    os.chdir(reldir)
+ 
+    config_map = []
+    from collections import namedtuple
+    confset = namedtuple('confset', ['config', 'os'])
+    artifacts = get_artifact_names(
+            f'{art_url_base}/{art_repo}',
+            artifact_prefixes)
+ 
+    # Download only the available artifacts that correspond to the
+    # requested build name into new release dir. 
+    for config in args.build_names:
+        for artifact in artifacts:
+            for prefix in artifact_prefixes:
+                if artifact == prefix+config+'.txt':
+                    aurl = f'{art_url_base}/{art_repo}/{artifact}'
+                    req = request.Request(aurl)
+                    result = request.urlopen(req)
+                    payload = result.readlines()
+
+                    # Determine OS of artifact and map to config name.
+                    if 'linux-64' in str(payload[-1]):
+                        cset = confset(config=config, os='Linux')
+                        config_map.append(cset)
+                    if 'osx-64' in str(payload[-1]):
+                        cset = confset(config=config, os='Macos')
+                        config_map.append(cset)
+
+                    # Replace jwst URL git hash with release tag name
+                    with open(artifact, 'w') as f:
+                        for line in payload:
+                            line = str(line.decode())
+                            line = modify_dep(line, release_tag)
+                            f.write(f'{line}\n')
+
+    write_readme(release_tag, config_map, 'README.md')
+
+    # Output the build_config to OS mappings.
+    for cset in config_map:
+        print(f"{cset.config}:{cset.os}")
+
+if __name__ == '__main__':
+    main()
+