diff options
Diffstat (limited to 'delivery_merge')
| -rw-r--r-- | delivery_merge/conda.py | 52 | ||||
| -rw-r--r-- | delivery_merge/merge.py | 39 | ||||
| -rw-r--r-- | delivery_merge/utils.py | 13 |
3 files changed, 98 insertions, 6 deletions
diff --git a/delivery_merge/conda.py b/delivery_merge/conda.py index 709dbaf..44e15de 100644 --- a/delivery_merge/conda.py +++ b/delivery_merge/conda.py @@ -9,12 +9,25 @@ from subprocess import run ENV_ORIG = os.environ.copy() +class BadPlatform(Exception): + pass + + def conda_installer(ver, prefix='./miniconda3'): + """ Install miniconda into a user-defined prefix and return its path + + :param ver: str: miniconda version (not conda version) + :param prefix: str: path to install miniconda into + :returns: str: absolute path to installation prefix + :raises delivery_merge.conda.BadPlatform: when platform check fails + :raises subprocess.CalledProcessError: via check_returncode method + """ assert isinstance(ver, str) assert isinstance(prefix, str) prefix = os.path.abspath(prefix) + # Is miniconda already installed? if os.path.exists(prefix): print(f'{prefix}: exists', file=sys.stderr) return prefix @@ -23,36 +36,58 @@ def conda_installer(ver, prefix='./miniconda3'): version = ver arch = 'x86_64' platform = sys.platform + + # Emit their installer's concept of "platform" if sys.platform == 'darwin': platform = 'MacOSX' elif sys.platform == 'linux': platform = 'Linux' + else: + raise BadPlatform(f'{sys.platform} is not supported.') url_root = 'https://repo.continuum.io/miniconda' installer = f'{name}-{version}-{platform}-{arch}.sh' url = f'{url_root}/{installer}' install_command = f'./{installer} -b -p {prefix}'.split() + # Download installer if not os.path.exists(installer): with requests.get(url, stream=True) as data: with open(installer, 'wb') as fd: for chunk in data.iter_content(chunk_size=16384): fd.write(chunk) os.chmod(installer, 0o755) - run(install_command) + + # Perform installation + run(install_command).check_returncode() return prefix def conda_init_path(prefix): + """ Redefines $PATH so subsequent shell calls use the just-installed + miniconda prefix. This function will not continue prepending to $PATH + so it's safe to call more than once. + + :param prefix: str: path to miniconda installation + :returns: None + """ if os.environ['PATH'] != ENV_ORIG['PATH']: os.environ['PATH'] = ENV_ORIG['PATH'] os.environ['PATH'] = ':'.join([os.path.join(prefix, 'bin'), os.environ['PATH']]) - print(f"PATH = {os.environ['PATH']}") def conda_activate(env_name): + """ Activate a conda environment + + Assume: `conda_init_path` as been called beforehand + Warning: Arbitrary code execution is possible here due to `shell` usage. + + :param env_name: str: conda environment to activate + :returns: dict: new runtime environment + :raises subprocess.CalledProcessError: via check_returncode method + """ proc = run(f"source activate {env_name} && env", capture_output=True, shell=True) @@ -62,6 +97,15 @@ def conda_activate(env_name): @contextmanager def conda_env_load(env_name): + """ A simple wrapper for `conda_activate` + The current runtime environment is replaced and restored + + >>> with conda_env_load('some_env') as _: + >>> # do something + + :param env_name: str: conda environment to activate + :returns: None + """ last = os.environ.copy() os.environ = conda_activate(env_name) try: @@ -71,4 +115,8 @@ def conda_env_load(env_name): def conda(*args): + """ Execute conda shell commands + + :returns: subprocess.CompletedProcess object + """ return sh('conda', *args) diff --git a/delivery_merge/merge.py b/delivery_merge/merge.py index 457c433..01b9732 100644 --- a/delivery_merge/merge.py +++ b/delivery_merge/merge.py @@ -11,17 +11,27 @@ DMFILE_RE = re.compile(r'^(?P<name>.*)[=<>~\!](?P<version>.*).*$') def comment_find(s, delims=[';', '#']): - """ Return index of first match + """ Find the first occurence of a comment in a string + + :param s: string + :param delims: list: of comment delimiters + :returns: integer: index of first match """ for delim in delims: - pos = s.find(delim) - if pos != -1: + index = s.find(delim) + if index != -1: break - return pos + return index def dmfile(filename): + """ Return the contents of a file without comments + + :param filename: string: path to file + :returns: list: data in file + """ + # TODO: Use DMFILE_RE here instead of `testable_packages` result = [] with open(filename, 'r') as fp: for line in fp: @@ -39,6 +49,15 @@ def dmfile(filename): def env_combine(filename, conda_env, conda_channels=[]): + """ Install packages listed in `filename` inside `conda_env`. + Packages are quote-escaped to prevent spurious file redirection. + + :param filename: str: path to file + :param conda_env: str: conda environment name + :param conda_channels: list: channel URLs + :returns: None + :raises subprocess.CalledProcessError: via check_returncode method + """ packages = [] channels_result = '--override-channels ' @@ -53,11 +72,16 @@ def env_combine(filename, conda_env, conda_channels=[]): conda_env, channels_result, packages_result) if proc.stderr: print(proc.stderr.decode()) + proc.check_returncode() def testable_packages(filename, prefix): """ Scan a mini/anaconda prefix for unpacked packages matching versions requested by dmfile. + + :param filename: str: path to file + :param prefix: str: path to conda root directory (aka prefix) + :returns: dict: git commit hash and repository URL information """ pkgdir = os.path.join(prefix, 'pkgs') paths = [] @@ -96,6 +120,13 @@ def testable_packages(filename, prefix): def integration_test(pkg_data, conda_env, results_root='.'): + """ + :param pkg_data: dict: data returned by `testable_packages` method + :param conda_env: str: conda environment name + :param results_root: str: path to store XML reports + :returns: None + :raises subprocess.CalledProcessError: via check_returncode method + """ results_root = os.path.abspath(os.path.join(results_root, 'results')) src_root = os.path.abspath('src') diff --git a/delivery_merge/utils.py b/delivery_merge/utils.py index 2344a13..8935dd7 100644 --- a/delivery_merge/utils.py +++ b/delivery_merge/utils.py @@ -4,6 +4,12 @@ from subprocess import run def sh(prog, *args): + """ Execute a program with arguments + :param prog: str: path to program + :param args: tuple: variadic arguments + Accepts any combination of strings passed as arguments + :returns: subprocess.CompletedProcess + """ command = [prog] tmp = [] for arg in args: @@ -15,11 +21,17 @@ def sh(prog, *args): def git(*args): + """ Execute git commands + :param args: tuple: variadic arguments to pass to git + :returns: subprocess.CompletedProcess + """ return sh('git', *args) def getenv(s): """ Convert string of key pairs to dictionary format + :param s: str: key pairs separated by newlines + :returns: dict: converted key pairs """ return dict([x.split('=', 1) for x in s.splitlines()]) @@ -27,6 +39,7 @@ def getenv(s): @contextmanager def pushd(path): """ Equivalent to shell pushd/popd behavior + :param path: str: path to directory """ last = os.path.abspath(os.getcwd()) os.chdir(path) |