#!/usr/bin/env python import contextlib import os import yaml import subprocess import sys from glob import glob from packaging.specifiers import SpecifierSet from packaging.version import Version, InvalidVersion USER_CONFIG = os.path.expanduser('~/.blast/config.yaml') USER_CONFIG_EXISTS = os.path.exists(USER_CONFIG) VERSION_BAD_PATTERNS = ['release_', '_release', 'release-', '-release'] SPECIFIERS = ['~', '!', '<', '>', '='] @contextlib.contextmanager def use_directory(path): prev = os.path.abspath(os.curdir) os.chdir(path) try: yield finally: os.chdir(prev) class PyEnv: def __init__(self, version, venv): self.version = version self.venv = venv self.environ = None def _cmd(self, *args, **kwargs): if not args: raise ValueError('Expecting arguments to pass to pyenv. ' 'Got nothing.') command = [] proc = None shell = False stdout = None stderr = None env = self.environ if kwargs.get('redirect', False): stdout = subprocess.PIPE stderr = subprocess.PIPE if kwargs.get('override', False): command = ['pyenv'] for arg in args: command.append(arg) if kwargs.get('shell', False): shell = True command = ' '.join(command) # convert to string try: proc = subprocess.run(command, env=env, shell=shell, stdout=stdout, stderr=stderr, check=True, encoding='utf-8') except subprocess.CalledProcessError as cpe: if cpe.returncode: print('Command (exit {}): {}'.format(cpe.returncode, command)) return cpe return proc def run(self, *args, **kwargs): proc = self._cmd(*args, **kwargs) return proc def create(self): if os.environ.get('PYENV_ROOT'): if not os.path.exists(os.path.join(os.environ['PYENV_ROOT'], 'versions', self.version)): proc_inst = self._cmd('install', '-s', self.version, override=True) if proc_inst.stderr: if 'already' not in proc_inst.stderr: print(proc_inst.stderr) exit(1) if not os.path.exists(os.path.join(os.environ['PYENV_ROOT'], 'versions', self.venv)): self._cmd('virtualenv', self.version, self.venv, override=True) else: raise RuntimeError('PYENV_ROOT is not defined.') exit(1) def activate(self): env = {} command = 'eval "$(pyenv init -)" ' \ '&& eval "$(pyenv virtualenv-init -)" ' \ '&& pyenv activate "{}" ' \ '&& pyenv rehash && printenv'.format(self.venv) proc = self._cmd(command, shell=True, redirect=True) for record in proc.stdout.splitlines(): if '=' not in record: continue k, v = record.split('=', 1) env[k] = v self.environ = env.copy() class Git: def __init__(self, uri, root='.'): self.uri = uri self.root = os.path.abspath(root) if not os.path.exists(self.root): print('Making {}'.format(self.root)) os.makedirs(self.root) self.base = os.path.basename(self.uri).replace('.git', '') self.basepath = os.path.abspath(os.path.join(self.root, self.base)) self._tags = [] def _cmd(self, *args, **kwargs): if not args: raise ValueError('Expecting arguments to pass to Git. ' 'Got nothing.') command = ['git'] proc = None for arg in args: command.append(arg) try: proc = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True, encoding='utf-8') except subprocess.CalledProcessError as cpe: print('Command (exit {}): {}'.format(cpe.returncode, command)) print(cpe.stderr.strip()) print(cpe) exit(1) return proc def _has_repo(self): if not os.path.exists('.git'): raise FileNotFoundError('No git repository present.') return True def clone(self): if os.path.exists(self.basepath): return self.basepath with use_directory(self.root): self._cmd('clone', self.uri) return self.basepath def fetch(self, *args, **kwargs): self._has_repo() self._cmd('fetch', *args, *kwargs) def checkout(self, ref, **kwargs): self._has_repo() self._cmd('checkout', ref, **kwargs) def reset(self, *args, **kwargs): self._has_repo() self._cmd('reset', *args, **kwargs) def clean(self, *args, **kwargs): self._has_repo() self._cmd('clean', *args, **kwargs) def describe(self, *args, **kwargs): self._has_repo() result = self._cmd('describe', *args, **kwargs) return result.stdout.strip() def tag_nearest(self): self._has_repo() return self.describe('--tags', '--abbrev=0') @property def tags(self): self._has_repo() tags = [x.strip() for x in self._cmd('tag', quiet=True).stdout.splitlines()] if len(self._tags) == len(tags): return self._tags self._tags = [] # Clear tags for tag in tags: self._tags.append(tag) return self._tags def setuptools_inject(): with open('setup.py', 'r') as script: orig = script.read().splitlines() orig.insert(0, 'import setuptools') with open('setup.py', 'w+') as script: new = '\n'.join(orig) script.write(new) def normalize_tag(tag, bad_patterns): # Normalize non-standard tagging conventions if bad_patterns is not None: for pattern in bad_patterns: tag = tag.replace(pattern, '') return tag def eval_specifier(spec, tag, bad_patterns=None): # Determine if specifiers are present in spec string have_specifier = False for ch in spec: if ch in SPECIFIERS: have_specifier = True # When no specifier is present we need to prepend one # to satisfy SpecifierSet's basic input requirements if not have_specifier: spec = '==' + spec spec = SpecifierSet(spec) tag = normalize_tag(tag) try: tag = Version(tag) except InvalidVersion as e: print("{}".format(e), file=sys.stderr) tag = '' return tag in spec def dist_exists(root, mode, bad_patterns, *hints): ext = '' if mode == 'sdist': ext = '.tar.gz' elif mode == 'bdist_wheel': ext = '.whl' else: raise NotImplementedError('Mode "{}" is not implemented.'.format(mode)) pattern = normalize_tag('*'.join(hints), bad_patterns) + '*' + ext paths = glob(os.path.join(root, pattern)) if paths: return True return False if __name__ == '__main__': # import argparse # from pprint import pprint # parser = argparse.ArgumentParser() # parser.add_argument('-c', '--config', action='store', default=USER_CONFIG) # parser.add_argument('repofile', action='store') config = yaml.load(""" host: https://github.com organization: spacetelescope global: python: - 3.5.5 - 3.6.6 - 3.7.0 requires: - wheel #version_latest: true version_bad_patterns: - 'release_' - '_release' - 'release-' - '-release' - 'v' - 'V' projects: acstools: requires: - relic asdf: requires: null calcos: requires: null costools: requires: null crds: requires: - astropy - numpy - requests setuptools_inject: true drizzle: requires: null drizzlepac: requires: - astropy - numpy fitsblender: requires: null imexam: requires: null nictools: requires: null pysynphot: requires: null reftools: requires: null relic: requires: null stistools: requires: null stsci.convolve: requires: null stsci.distutils: requires: null stsci.image: requires: null stsci.imagemanip: requires: null stsci.imagestats: requires: null stsci.ndimage: requires: null stsci.numdisplay: requires: null stsci.stimage: requires: null stsci.skypac: requires: null stsci.sphere: requires: null stsci.tools: requires: null stregion: requires: null stwcs: requires: null wfpc2tools: requires: null wfc3tools: requires: null """) output_dir = os.path.abspath(os.path.join(os.curdir, 'upload')) if config.get('output_dir'): output_dir = os.path.abspath(config['output_dir']) if not os.path.exists(output_dir): os.mkdir(output_dir) # Begin aliveness checks before we get too far check_keys = ['global', 'host', 'organization', 'projects'] failed_keys = False failed_requires = False # Check general configuration keys for key in check_keys: if not config.get(key): print('Error: The `{}` key is required.'.format(key), file=sys.stderr) failed_keys = True if failed_keys: exit(1) # Check structure of project dictionaries for project, info in config['projects'].items(): if info is None and not info.get('requires'): print('Error: {}: Missing `requires` list'.format(project), file=sys.stderr) failed_requires = True if failed_requires: exit(1) # Perform matrix tasks for project, info in config['projects'].items(): url = '/'.join([config['host'], config['organization'], project]) repo = Git(url, root='src') print('Repository: {}'.format(url)) print('Source directory: {}'.format(repo.basepath)) try: repo.clone() except FileNotFoundError as e: print('Skipping "{}" due to: {}'.format(project, e)) continue with use_directory(repo.basepath): repo.fetch('--all', '--tags') for python_version in config['global']['python']: tags = [] venv_name = 'py{}'.format(python_version.replace('.', '')) print('=> Using Python {} (virtual: {})...'.format(python_version, venv_name)) pyenv = PyEnv(python_version, venv_name) pyenv.create() pyenv.activate() # Upgrade PIP pyenv.run('pip', 'install', '-q', '--upgrade', 'pip', redirect=True) # Generic setup if config.get('global'): if config['global'].get('requires'): for pkg in config['global']['requires']: pyenv.run('pip', 'install', '-q', pkg, redirect=True) if config['global'].get('version_latest'): repo.reset('--hard') repo.clean('-f', '-x', '-d') repo.checkout('master') tags = [repo.tag_nearest()] else: tags = repo.tags if info.get('requires'): for pkg in info['requires']: pyenv.run('pip', 'install', '-q', pkg) for tag in tags: sanitize = VERSION_BAD_PATTERNS if info.get('build_versions'): spec = info['build_versions'] if config['global'].get('version_bad_patterns'): sanitize += config['global']['version_bad_patterns'] if info.get('version_bad_patterns'): sanitize += info['version_bad_patterns'] if not eval_specifier(spec, tag, sanitize): continue print('==> Building {}::{}'.format(project, tag)) repo.reset('--hard') repo.clean('-f', '-x', '-d') repo.checkout(tag) if info.get('setuptools_inject', False): print('===> Overriding distutils with setuptools') setuptools_inject() for build_command, build_args in [('sdist', []), ('bdist_wheel', [])]: print('===> {}::{}: {}: '.format(project, tag, build_command), end='') if not dist_exists(output_dir, build_command, sanitize, project, tag): proc = pyenv.run('python', 'setup.py', build_command, '-d', output_dir, *build_args, redirect=True) if proc.stderr and proc.returncode: print("FAILED") print('#{}'.format('!' * 78)) print(proc.stderr) print('#{}'.format('!' * 78)) elif not proc.returncode: print("SUCCESS") else: print("EXISTS")