aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--.gitignore8
-rw-r--r--delivery_merge/__init__.py3
-rw-r--r--delivery_merge/cli/__init__.py0
-rw-r--r--delivery_merge/cli/merge.py56
-rw-r--r--delivery_merge/conda.py81
-rw-r--r--delivery_merge/git.py11
-rw-r--r--delivery_merge/merge.py119
-rw-r--r--delivery_merge/utils.py33
-rw-r--r--setup.py16
9 files changed, 327 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..0e0c0fd
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,8 @@
+*.egg*
+*.sh
+*.sk
+__pycache__
+results
+src
+miniconda*
+anaconda*
diff --git a/delivery_merge/__init__.py b/delivery_merge/__init__.py
new file mode 100644
index 0000000..60cb5a6
--- /dev/null
+++ b/delivery_merge/__init__.py
@@ -0,0 +1,3 @@
+
+import re
+from .cli import *
diff --git a/delivery_merge/cli/__init__.py b/delivery_merge/cli/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/delivery_merge/cli/__init__.py
diff --git a/delivery_merge/cli/merge.py b/delivery_merge/cli/merge.py
new file mode 100644
index 0000000..b0f581d
--- /dev/null
+++ b/delivery_merge/cli/merge.py
@@ -0,0 +1,56 @@
+import os
+from ..conda import conda, conda_installer, conda_init_path
+from ..merge import env_combine, testable_packages, integration_test
+from argparse import ArgumentParser
+
+
+def main():
+ parser = ArgumentParser()
+ parser.add_argument('--env-name', default='delivery',
+ help='name of environment')
+ parser.add_argument('--installer-version', required=True,
+ help='miniconda3 installer version')
+ parser.add_argument('--run-tests', action='store_true')
+ parser.add_argument('--dmfile', required=True)
+ parser.add_argument('base_spec')
+ args = parser.parse_args()
+
+ name = args.env_name
+ base_spec = args.base_spec
+ dmfile = args.dmfile
+ channels = ['https://astroconda.org/channel/main',
+ 'defaults',
+ 'http://ssb.stsci.edu/conda-dev']
+ delivery_root = 'delivery'
+ yamlfile = os.path.join(delivery_root, name + '.yml')
+ specfile = os.path.join(delivery_root, name + '.txt')
+
+ if not os.path.exists(delivery_root):
+ os.mkdir(delivery_root, 0o755)
+
+ prefix = conda_installer(args.installer_version)
+ conda_init_path(prefix)
+
+ if not os.path.exists(os.path.join(prefix, 'envs', name)):
+ print(f"Creating environment {name}...")
+ proc = conda('create', '-q', '-y', '-n', name, '--file', base_spec)
+ if proc.stderr:
+ print(proc.stderr.decode())
+
+ print(f"Merging requested packages into environment: {name}")
+ env_combine(dmfile, name, channels)
+
+ print("Exporting yaml configuration...")
+ conda('env', 'export', '-n', name, '--file', yamlfile)
+
+ print("Exporting explicit dump...")
+ with open(specfile, 'w+') as spec:
+ proc = conda('list', '--explicit', '-n', name)
+ spec.write(proc.stdout.decode())
+
+ if args.run_tests:
+ for package in testable_packages(args.dmfile, prefix):
+ print(f"Running tests: {package}")
+ integration_test(package, name)
+
+ print("Done!")
diff --git a/delivery_merge/conda.py b/delivery_merge/conda.py
new file mode 100644
index 0000000..57d715e
--- /dev/null
+++ b/delivery_merge/conda.py
@@ -0,0 +1,81 @@
+import os
+import requests
+import sys
+from .utils import getenv, sh
+from contextlib import contextmanager
+from subprocess import run
+
+
+ENV_ORIG = os.environ.copy()
+
+
+def conda_installer(ver, prefix='./miniconda3'):
+ assert isinstance(ver, str)
+ assert isinstance(prefix, str)
+
+ prefix = os.path.abspath(prefix)
+
+ if os.path.exists(prefix):
+ print(f'{prefix}: exists', file=sys.stderr)
+ return prefix
+
+ name = 'Miniconda3'
+ version = ver
+ arch = 'x86_64'
+ platform = sys.platform
+ if sys.platform == 'darwin':
+ platform = 'MacOSX'
+ elif sys.platform == 'linux':
+ platform = 'Linux'
+
+ url_root = 'https://repo.continuum.io/miniconda'
+ installer = f'{name}-{version}-{platform}-{arch}.sh'
+ url = f'{url_root}/{installer}'
+ install_command = f'./{installer} -b -p {prefix}'.split()
+
+ if not os.path.exists(installer):
+ with requests.get(url, stream=True) as data:
+ with open(installer, 'wb') as fd:
+ for chunk in data.iter_content(chunk_size=16384):
+ fd.write(chunk)
+ os.chmod(installer, 0o755)
+ run(install_command)
+
+ return prefix
+
+
+def conda_init_path(prefix):
+ if os.environ['PATH'] != ENV_ORIG['PATH']:
+ os.environ['PATH'] = ENV_ORIG['PATH']
+ os.environ['PATH'] = ':'.join([os.path.join(prefix, 'bin'),
+ os.environ['PATH']])
+ print(f"PATH = {os.environ['PATH']}")
+
+
+def conda_activate(env_name):
+ proc = run(f"source activate {env_name} && env",
+ capture_output=True,
+ shell=True)
+ proc.check_returncode()
+ return getenv(proc.stdout.decode()).copy()
+
+
+@contextmanager
+def conda_env_load(env_name):
+ last = os.environ.copy()
+ os.environ = conda_activate(env_name)
+ try:
+ yield
+ finally:
+ os.environ = last.copy()
+
+
+def conda(*args):
+ command = ['conda']
+ tmp = []
+ for arg in args:
+ tmp += arg.split()
+
+ command += tmp
+ print(f'Running: {" ".join(command)}')
+ return run(command, capture_output=True)
diff --git a/delivery_merge/git.py b/delivery_merge/git.py
new file mode 100644
index 0000000..9e8aa6a
--- /dev/null
+++ b/delivery_merge/git.py
@@ -0,0 +1,11 @@
+from subprocess import run
+
+def git(*args):
+ command = ['git']
+ tmp = []
+ for arg in args:
+ tmp += arg.split()
+
+ command += tmp
+ print(f'Running: {" ".join(command)}')
+ return run(command, capture_output=True)
diff --git a/delivery_merge/merge.py b/delivery_merge/merge.py
new file mode 100644
index 0000000..e8a9ba1
--- /dev/null
+++ b/delivery_merge/merge.py
@@ -0,0 +1,119 @@
+import os
+import re
+import yaml
+from .conda import conda, conda_env_load
+from .git import git
+from .utils import pushd
+from glob import glob
+from subprocess import run
+
+
+DMFILE_RE = re.compile(r'^(?P<name>.*)[=<>~\!](?P<version>.*).*$')
+
+
+def comment_find(s, delims=[';', '#']):
+ """ Return index of first match
+ """
+ for delim in delims:
+ pos = s.find(delim)
+ if pos != -1:
+ break
+
+ return pos
+
+
+def dmfile(filename):
+ result = []
+ with open(filename, 'r') as fp:
+ for line in fp:
+ line = line.strip()
+ comment_pos = comment_find(line)
+
+ if comment_pos >= 0:
+ line = line[:comment_pos]
+
+ if not line:
+ continue
+
+ result.append(line)
+ return result
+
+
+def env_combine(filename, conda_env, conda_channels=[]):
+ packages = []
+ channels_result = '--override-channels '
+
+ for line in dmfile(filename):
+ packages.append(f"'{line}'")
+
+ for channel in conda_channels:
+ channels_result += f'-c {channel} '
+
+ packages_result = ' '.join([x for x in packages])
+ proc = conda('install', '-q', '-y', '-n',
+ conda_env, channels_result, packages_result)
+ if proc.stderr:
+ print(proc.stderr.decode())
+
+
+def testable_packages(filename, prefix):
+ """ Scan a mini/anaconda prefix for unpacked packages matching versions
+ requested by dmfile.
+ """
+ pkgdir = os.path.join(prefix, 'pkgs')
+ paths = []
+
+ for line in dmfile(filename):
+ match = DMFILE_RE.match(line)
+ if match:
+ pkg = match.groupdict()
+
+ # Reconstruct ${package}-${version} format from
+ # ${package}${specifier}${version}
+ pattern = f"{pkg['name']}-{pkg['version']}*"
+
+ # Record path to extracted package
+ path = ''.join([x for x in glob(os.path.join(pkgdir, pattern))
+ if os.path.isdir(x)])
+ paths.append(path)
+
+ for root in paths:
+ info_d = os.path.join(root, 'info')
+ recipe_d = os.path.join(info_d, 'recipe')
+ git_log = os.path.join(info_d, 'git')
+
+ if not os.path.exists(git_log):
+ continue
+
+ with open(os.path.join(recipe_d, 'meta.yaml')) as yaml_data:
+ source = yaml.load(yaml_data.read())['source']
+
+ if not isinstance(source, dict):
+ continue
+
+ repository = source['git_url']
+ head = open(git_log).readlines()[1].split()[1]
+ yield dict(repo=repository, commit=head)
+
+
+def integration_test(pkg_data, conda_env, results_root='.'):
+ results_root = os.path.abspath(os.path.join(results_root, 'results'))
+ src_root = os.path.abspath('src')
+
+ if not os.path.exists(src_root):
+ os.mkdir(src_root, 0o755)
+
+ with pushd(src_root) as _:
+ repo_root = os.path.basename(pkg_data['repo']).replace('.git', '')
+ if not os.path.exists(repo_root):
+ git(f"clone --recursive {pkg_data['repo']} {repo_root}")
+
+ with pushd(repo_root) as _:
+ git(f"checkout {pkg_data['commit']}")
+
+ with conda_env_load(conda_env):
+ results = os.path.abspath(os.path.join(results_root,
+ repo_root,
+ 'result.xml'))
+ run("pip install -e .[test]".split()).check_returncode()
+ run(f"pytest -v --junitxml={results}".split(), check=True)
diff --git a/delivery_merge/utils.py b/delivery_merge/utils.py
new file mode 100644
index 0000000..1dca224
--- /dev/null
+++ b/delivery_merge/utils.py
@@ -0,0 +1,33 @@
+import os
+from contextlib import contextmanager
+from subprocess import run
+
+
+def sh(prog, *args):
+ command = [prog]
+ tmp = []
+ for arg in args:
+ tmp += arg.split()
+
+ command += tmp
+ print(f'Running: {" ".join(command)}')
+ return run(command, capture_output=True)
+
+
+def getenv(s):
+ """ Convert string of key pairs to dictionary format
+ """
+ return dict([x.split('=', 1) for x in s.splitlines()])
+
+
+@contextmanager
+def pushd(path):
+ """ Equivalent to shell pushd/popd behavior
+ """
+ last = os.path.abspath(os.getcwd())
+ os.chdir(path)
+ try:
+ yield
+ finally:
+ os.chdir(last)
+
diff --git a/setup.py b/setup.py
new file mode 100644
index 0000000..13b2c99
--- /dev/null
+++ b/setup.py
@@ -0,0 +1,16 @@
+from setuptools import setup, find_packages
+
+setup(
+ name='delivery_merge',
+ version='0.0.1',
+ packages=find_packages(),
+ entry_points={
+ 'console_scripts': [
+ 'delivery_merge=delivery_merge.cli.merge:main',
+ ],
+ },
+ install_requires=[
+ 'requests',
+ 'pyyaml',
+ ]
+)
generated by cgit (git 2.34.1) at 2025-04-28 13:17:43 -0400