From f293e7d7c76f383eae64f224834815f1b14a78e6 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Mon, 29 Jun 2015 00:29:14 -0400 Subject: Getting closer to the real deal --- cbc/environment.py | 2 +- cbc/meta.py | 26 ++++++++++---- cbc/utils.py | 42 ++++++++++++++++++++-- cbcbuild.py | 103 ++++++++++++++++++++++++++++++++++------------------- 4 files changed, 127 insertions(+), 46 deletions(-) mode change 100644 => 100755 cbcbuild.py diff --git a/cbc/environment.py b/cbc/environment.py index 7bc427c..a1c6b6e 100644 --- a/cbc/environment.py +++ b/cbc/environment.py @@ -21,7 +21,7 @@ class Environment(object): self.cbchome = self.environ['CBC_HOME'] if self.cbchome is None: - raise IncompleteEnv('Environment.cbchome is undefined') + raise IncompleteEnv('CBC_HOME is undefined.') self.cbchome = os.path.abspath(self.cbchome) diff --git a/cbc/meta.py b/cbc/meta.py index ed1baaa..7203726 100644 --- a/cbc/meta.py +++ b/cbc/meta.py @@ -24,7 +24,7 @@ class MetaData(object): raise MetaDataError('Expecting instance of cbc.environment.Environment, got: "{0}"'.format(type(env))) self.env = env - self.keywords = ['cbc_build', 'cbc_cgi'] + self.builtins = ['cbc_build', 'cbc_cgi'] self.fields = self.convert_conda_fields(conda_build.metadata.FIELDS) @@ -35,33 +35,45 @@ class MetaData(object): # Include user-defined build fields self.config.read(self.filename) - # Convert ConfigParser -> dict + # Convert ConfigParser -> generic dictionary self.local = self.as_dict(self.config) #if not self.local['requirements']['build']: # raise MetaDataError('Incomplete or missing "requirements" section: self.local[\'requirements\'] ', self.local['requirements']['build']) # Convert requirements to lists - for section in self.local['requirements'].keys(): - self.local['requirements'][section] = self.config.getlist('requirements', section) + #for section in self.local['requirements'].keys(): + # self.local['requirements'][section] = self.config.getlist('requirements', section) + + #Field list conversion table taken from conda_build.metadata: + for field in ('source/patches', + 'build/entry_points', 'build/script_env', + 'build/features', 'build/track_features', + 'requirements/build', 'requirements/run', + 'requirements/conflicts', 'test/requires', + 'test/files', 'test/commands', 'test/imports'): + section, key = field.split('/') + if self.local[section][key]: + self.local[section][key] = self.config.getlist(section, key) self.local_metadata = {} - for keyword in self.keywords: + for keyword in self.builtins: if keyword in self.local: self.local_metadata[keyword] = self.local[keyword] # Convert dict to YAML-compatible dict - self.conda_metadata = self.scrub(self.local, self.keywords) + self.conda_metadata = self.scrub(self.local, self.builtins) def run(self): self.render_scripts() def render_scripts(self): + '''Write all conda scripts + ''' for maskkey, maskval in self.env.config['script'].items(): for metakey, metaval in self.compile().items(): if metakey in maskkey: with open(maskval, 'w+') as metafile: - print("Writing: {0}".format(maskval)) metafile.write(metaval) diff --git a/cbc/utils.py b/cbc/utils.py index 096e156..e276566 100644 --- a/cbc/utils.py +++ b/cbc/utils.py @@ -1,4 +1,6 @@ -from subprocess import Popen, PIPE, check_output, CalledProcessError +from .meta import MetaData +from .exceptions import CondaBuildError +from subprocess import Popen, PIPE, STDOUT, check_call, check_output, CalledProcessError def conda_search(pkgname): @@ -27,4 +29,40 @@ def conda_install(pkgname): print(line) except CalledProcessError as cpe: print('{0}\nexit={1}'.format(' '.join(cpe.cmd), cpe.returncode)) - \ No newline at end of file + + +def conda_reinstall(pkgname): + # Until I can figure out a good way to build with the conda API + # we'll use the CLI interface: + commands = ['conda remove --yes {0}'.format(pkgname).split(), + 'conda install --use-local --yes {0}'.format(pkgname).split()] + for command in commands: + try: + for line in (check_output(command).decode('utf-8').splitlines()): + print(line) + except CalledProcessError as cpe: + print('{0}\nexit={1}'.format(' '.join(cpe.cmd), cpe.returncode)) + + +def conda_builder(metadata, args): + if not isinstance(metadata, MetaData): + raise CondaBuildError('Expecting instance of conda_build.metadata.MetaData, got: "{0}"'.format(type(metadata))) + + bad_egg = 'UNKNOWN.egg-info' + command = ['conda', 'build', metadata.env.pkgdir ] + + try: + for line in (check_output(command).decode('utf-8').splitlines()): + if line.startswith('#'): + continue + print(line) + if bad_egg in line: + raise CondaBuildError('Bad setuptools metadata produced UNKNOWN.egg-info instead of {0}*.egg-info!'.format(metadata.local['package']['name'])) + #OK Conda, let's play rough. stdout/stderr only, no exit for you. + except SystemExit: + print('Discarding SystemExit issued by setuptools') + except CalledProcessError as cpe: + print(cpe) + return False + + return True \ No newline at end of file diff --git a/cbcbuild.py b/cbcbuild.py old mode 100644 new mode 100755 index 35453e5..92d3646 --- a/cbcbuild.py +++ b/cbcbuild.py @@ -1,47 +1,78 @@ +#!/usr/bin/env python +import argparse import os +import sys import cbc -import cbc.server -import conda_build import conda_build.metadata -import conda_build.build -import argparse -import time -from cbc.exceptions import CondaBuildError -parser = argparse.ArgumentParser() -parser.add_argument('cbcfile', action='store', nargs='*', help='Build configuration file') -args = parser.parse_args() + + os.environ['CBC_HOME'] = 'tests/data/build' -env = cbc.environment.Environment() +#sys.argv.append('--force-rebuild') +#sys.argv.append('tests/data/aprio.ini') -# Convert cbcfile paths to absolute paths -args.cbcfile = [ os.path.abspath(x) for x in args.cbcfile ] -# Verify we have a file that exists -for cbcfile in args.cbcfile: - if not os.path.exists(cbcfile): - print('{} does not exist.'.format(cbcfile)) - exit(1) - elif not os.path.isfile(cbcfile): - print('{} is not a file.'.format(cbcfile)) - exit(1) - -# Perform build(s) -for cbcfile in args.cbcfile: - # Ensure the working directory remains the same throughout. - os.chdir(env.pwd) - - metadata = cbc.meta.MetaData(cbcfile, env) - metadata.env.mkpkgdir(metadata.local['package']['name']) - metadata.render_scripts() +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--force-rebuild', + action='store_true', + help='Do not stop if package already installed') + parser.add_argument('cbcfile', + nargs='+', + help='CBC metadata') + args = parser.parse_args() - conda_metadata = conda_build.metadata.MetaData(env.pkgdir) + # Initialize internal environment + env = cbc.environment.Environment() - if cbc.utils.conda_search(conda_metadata.name()) == conda_metadata.dist(): - print('{0} metadata matches an installed package; increment the build number to rebuild.'.format(conda_metadata.dist())) - continue + # Convert cbcfile paths to absolute paths + args.cbcfile = [ os.path.abspath(x) for x in args.cbcfile ] - conda_build.build.build(conda_metadata, get_src=True, verbose=True) - cbc.utils.conda_install(conda_metadata.name()) - \ No newline at end of file + # Verify we have a file that exists + for cbcfile in args.cbcfile: + if not os.path.exists(cbcfile): + print('{} does not exist.'.format(cbcfile)) + exit(1) + elif not os.path.isfile(cbcfile): + print('{} is not a file.'.format(cbcfile)) + exit(1) + + # Perform build(s) + for cbcfile in args.cbcfile: + print('Using cbc build configuration: {0}'.format(cbcfile)) + # Ensure the working directory remains the same throughout. + os.chdir(env.pwd) + + metadata = cbc.meta.MetaData(cbcfile, env) + metadata.env.mkpkgdir(metadata.local['package']['name']) + metadata.render_scripts() + + conda_metadata = conda_build.metadata.MetaData(env.pkgdir) + + if not args.force_rebuild: + if cbc.utils.conda_search(conda_metadata.name()) == conda_metadata.dist(): + print('{0} metadata matches an installed package; increment the build number to rebuild or use --force-rebuild.'.format(conda_metadata.dist())) + continue + + conda_builder_args = {'get_src': True, 'verbose': False} + + try: + built = cbc.utils.conda_builder(metadata, conda_builder_args) + if not built: + print('Failure occurred while building: {0}'.format(conda_metadata.dist())) + continue + except cbc.exceptions.CondaBuildError as cbe: + print(cbe) + continue + + package_exists = cbc.utils.conda_search(conda_metadata.name()) + if not package_exists: + cbc.utils.conda_install(conda_metadata.name()) + elif package_exists and args.force_rebuild: + cbc.utils.conda_reinstall(conda_metadata.name()) + + print('') + + + \ No newline at end of file -- cgit