7 files changed, 303 insertions, 0 deletions
diff --git a/delivery_merge/__init__.py b/delivery_merge/__init__.py
new file mode 100644
index 0000000..60cb5a6
--- /dev/null
+++ b/delivery_merge/__init__.py
@@ -0,0 +1,3 @@
+
+import re
+from .cli import *
diff --git a/delivery_merge/cli/__init__.py b/delivery_merge/cli/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/delivery_merge/cli/__init__.py
diff --git a/delivery_merge/cli/merge.py b/delivery_merge/cli/merge.py
new file mode 100644
index 0000000..b0f581d
--- /dev/null
+++ b/delivery_merge/cli/merge.py
@@ -0,0 +1,56 @@
+import os
+from ..conda import conda, conda_installer, conda_init_path
+from ..merge import env_combine, testable_packages, integration_test
+from argparse import ArgumentParser
+
+
+def main():
+    parser = ArgumentParser()
+    parser.add_argument('--env-name', default='delivery',
+                        help='name of environment')
+    parser.add_argument('--installer-version', required=True,
+                        help='miniconda3 installer version')
+    parser.add_argument('--run-tests', action='store_true')
+    parser.add_argument('--dmfile', required=True)
+    parser.add_argument('base_spec')
+    args = parser.parse_args()
+
+    name = args.env_name
+    base_spec = args.base_spec
+    dmfile = args.dmfile
+    channels = ['https://astroconda.org/channel/main',
+                'defaults',
+                'http://ssb.stsci.edu/conda-dev']
+    delivery_root = 'delivery'
+    yamlfile = os.path.join(delivery_root, name + '.yml')
+    specfile = os.path.join(delivery_root, name + '.txt')
+
+    if not os.path.exists(delivery_root):
+        os.mkdir(delivery_root, 0o755)
+
+    prefix = conda_installer(args.installer_version)
+    conda_init_path(prefix)
+
+    if not os.path.exists(os.path.join(prefix, 'envs', name)):
+        print(f"Creating environment {name}...")
+        proc = conda('create', '-q', '-y', '-n',  name, '--file', base_spec)
+        if proc.stderr:
+            print(proc.stderr.decode())
+
+    print(f"Merging requested packages into environment: {name}")
+    env_combine(dmfile, name, channels)
+
+    print("Exporting yaml configuration...")
+    conda('env', 'export', '-n', name, '--file', yamlfile)
+
+    print("Exporting explicit dump...")
+    with open(specfile, 'w+') as spec:
+        proc = conda('list', '--explicit', '-n', name)
+        spec.write(proc.stdout.decode())
+
+    if args.run_tests:
+        for package in testable_packages(args.dmfile, prefix):
+            print(f"Running tests: {package}")
+            integration_test(package, name)
+
+    print("Done!")
diff --git a/delivery_merge/conda.py b/delivery_merge/conda.py
new file mode 100644
index 0000000..57d715e
--- /dev/null
+++ b/delivery_merge/conda.py
@@ -0,0 +1,81 @@
+import os
+import requests
+import sys
+from .utils import getenv, sh
+from contextlib import contextmanager
+from subprocess import run
+
+
+ENV_ORIG = os.environ.copy()
+
+
+def conda_installer(ver, prefix='./miniconda3'):
+    assert isinstance(ver, str)
+    assert isinstance(prefix, str)
+
+    prefix = os.path.abspath(prefix)
+
+    if os.path.exists(prefix):
+        print(f'{prefix}: exists', file=sys.stderr)
+        return prefix
+
+    name = 'Miniconda3'
+    version = ver
+    arch = 'x86_64'
+    platform = sys.platform
+    if sys.platform == 'darwin':
+        platform = 'MacOSX'
+    elif sys.platform == 'linux':
+        platform = 'Linux'
+
+    url_root = 'https://repo.continuum.io/miniconda'
+    installer = f'{name}-{version}-{platform}-{arch}.sh'
+    url = f'{url_root}/{installer}'
+    install_command = f'./{installer} -b -p {prefix}'.split()
+
+    if not os.path.exists(installer):
+        with requests.get(url, stream=True) as data:
+            with open(installer, 'wb') as fd:
+                for chunk in data.iter_content(chunk_size=16384):
+                    fd.write(chunk)
+        os.chmod(installer, 0o755)
+    run(install_command)
+
+    return prefix
+
+
+def conda_init_path(prefix):
+    if os.environ['PATH'] != ENV_ORIG['PATH']:
+        os.environ['PATH'] = ENV_ORIG['PATH']
+    os.environ['PATH'] = ':'.join([os.path.join(prefix, 'bin'),
+                                   os.environ['PATH']])
+    print(f"PATH = {os.environ['PATH']}")
+
+
+def conda_activate(env_name):
+    proc = run(f"source activate {env_name} && env",
+               capture_output=True,
+               shell=True)
+    proc.check_returncode()
+    return getenv(proc.stdout.decode()).copy()
+
+
+@contextmanager
+def conda_env_load(env_name):
+    last = os.environ.copy()
+    os.environ = conda_activate(env_name)
+    try:
+        yield
+    finally:
+        os.environ = last.copy()
+
+
+def conda(*args):
+    command = ['conda']
+    tmp = []
+    for arg in args:
+        tmp += arg.split()
+
+    command += tmp
+    print(f'Running: {" ".join(command)}')
+    return run(command, capture_output=True)
diff --git a/delivery_merge/git.py b/delivery_merge/git.py
new file mode 100644
index 0000000..9e8aa6a
--- /dev/null
+++ b/delivery_merge/git.py
@@ -0,0 +1,11 @@
+from subprocess import run
+
+def git(*args):
+    command = ['git']
+    tmp = []
+    for arg in args:
+        tmp += arg.split()
+
+    command += tmp
+    print(f'Running: {" ".join(command)}')
+    return run(command, capture_output=True)
diff --git a/delivery_merge/merge.py b/delivery_merge/merge.py
new file mode 100644
index 0000000..e8a9ba1
--- /dev/null
+++ b/delivery_merge/merge.py
@@ -0,0 +1,119 @@
+import os
+import re
+import yaml
+from .conda import conda, conda_env_load
+from .git import git
+from .utils import pushd
+from glob import glob
+from subprocess import run
+
+
+DMFILE_RE = re.compile(r'^(?P<name>.*)[=<>~\!](?P<version>.*).*$')
+
+
+def comment_find(s, delims=[';', '#']):
+    """ Return index of first match
+    """
+    for delim in delims:
+        pos = s.find(delim)
+        if pos != -1:
+            break
+
+    return pos
+
+
+def dmfile(filename):
+    result = []
+    with open(filename, 'r') as fp:
+        for line in fp:
+            line = line.strip()
+            comment_pos = comment_find(line)
+
+            if comment_pos >= 0:
+                line = line[:comment_pos]
+
+            if not line:
+                continue
+
+            result.append(line)
+    return result
+
+
+def env_combine(filename, conda_env, conda_channels=[]):
+    packages = []
+    channels_result = '--override-channels '
+
+    for line in dmfile(filename):
+        packages.append(f"'{line}'")
+
+    for channel in conda_channels:
+        channels_result += f'-c {channel} '
+
+    packages_result = ' '.join([x for x in packages])
+    proc = conda('install', '-q', '-y', '-n',
+                 conda_env, channels_result, packages_result)
+    if proc.stderr:
+        print(proc.stderr.decode())
+
+
+def testable_packages(filename, prefix):
+    """ Scan a mini/anaconda prefix for unpacked packages matching versions
+    requested by dmfile.
+    """
+    pkgdir = os.path.join(prefix, 'pkgs')
+    paths = []
+
+    for line in dmfile(filename):
+        match = DMFILE_RE.match(line)
+        if match:
+            pkg = match.groupdict()
+
+            # Reconstruct ${package}-${version} format from
+            # ${package}${specifier}${version}
+            pattern = f"{pkg['name']}-{pkg['version']}*"
+
+            # Record path to extracted package
+            path = ''.join([x for x in glob(os.path.join(pkgdir, pattern))
+                            if os.path.isdir(x)])
+            paths.append(path)
+
+    for root in paths:
+        info_d = os.path.join(root, 'info')
+        recipe_d = os.path.join(info_d, 'recipe')
+        git_log = os.path.join(info_d, 'git')
+
+        if not os.path.exists(git_log):
+            continue
+
+        with open(os.path.join(recipe_d, 'meta.yaml')) as yaml_data:
+            source = yaml.load(yaml_data.read())['source']
+
+        if not isinstance(source, dict):
+            continue
+
+        repository = source['git_url']
+        head = open(git_log).readlines()[1].split()[1]
+        yield dict(repo=repository, commit=head)
+
+
+def integration_test(pkg_data, conda_env, results_root='.'):
+    results_root = os.path.abspath(os.path.join(results_root, 'results'))
+    src_root = os.path.abspath('src')
+
+    if not os.path.exists(src_root):
+        os.mkdir(src_root, 0o755)
+
+    with pushd(src_root) as _:
+        repo_root = os.path.basename(pkg_data['repo']).replace('.git', '')
+        if not os.path.exists(repo_root):
+            git(f"clone --recursive {pkg_data['repo']} {repo_root}")
+
+        with pushd(repo_root) as _:
+            git(f"checkout {pkg_data['commit']}")
+
+            with conda_env_load(conda_env):
+                results = os.path.abspath(os.path.join(results_root,
+                                                       repo_root,
+                                                       'result.xml'))
+                run("pip install -e .[test]".split()).check_returncode()
+                run(f"pytest -v --junitxml={results}".split(), check=True)
diff --git a/delivery_merge/utils.py b/delivery_merge/utils.py
new file mode 100644
index 0000000..1dca224
--- /dev/null
+++ b/delivery_merge/utils.py
@@ -0,0 +1,33 @@
+import os
+from contextlib import contextmanager
+from subprocess import run
+
+
+def sh(prog, *args):
+    command = [prog]
+    tmp = []
+    for arg in args:
+        tmp += arg.split()
+
+    command += tmp
+    print(f'Running: {" ".join(command)}')
+    return run(command, capture_output=True)
+
+
+def getenv(s):
+    """ Convert string of key pairs to dictionary format
+    """
+    return dict([x.split('=', 1) for x in s.splitlines()])
+
+
+@contextmanager
+def pushd(path):
+    """ Equivalent to shell pushd/popd behavior
+    """
+    last = os.path.abspath(os.getcwd())
+    os.chdir(path)
+    try:
+        yield
+    finally:
+        os.chdir(last)
+