From 237d2159faac7dadb0232e35f7016588db568808 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Fri, 17 Jul 2015 13:57:23 -0400 Subject: Restructuring and setuptools integration --- .gitignore | 7 +++- MANIFEST.in | 1 + cbc/__init__.py | 2 +- cbc/cli/__init__.py | 0 cbc/cli/build.py | 100 ++++++++++++++++++++++++++++++++++++++++++++++ cbc/cli/server.py | 53 ++++++++++++++++++++++++ cbc/server.py | 53 ------------------------ cbc/tests/__init__.py | 0 cbc/tests/data/test.ini | 47 ++++++++++++++++++++++ cbc/tests/test_cbc.py | 66 ++++++++++++++++++++++++++++++ cbcbuild.py | 101 ---------------------------------------------- setup.py | 34 ++++++++++++++++ tests/data/test.ini | 47 ---------------------- tests/test.py | 70 -------------------------------- version.py | 104 ++++++++++++++++++++++++++++++++++++++++++++++++ 15 files changed, 412 insertions(+), 273 deletions(-) create mode 100644 MANIFEST.in create mode 100644 cbc/cli/__init__.py create mode 100755 cbc/cli/build.py create mode 100644 cbc/cli/server.py delete mode 100644 cbc/server.py create mode 100644 cbc/tests/__init__.py create mode 100644 cbc/tests/data/test.ini create mode 100644 cbc/tests/test_cbc.py delete mode 100755 cbcbuild.py create mode 100644 setup.py delete mode 100644 tests/data/test.ini delete mode 100644 tests/test.py create mode 100644 version.py diff --git a/.gitignore b/.gitignore index d8a6d24..77f4530 100755 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,9 @@ *.pyc *.DS_Store *pycache* -test/output +*.egg-info +dist/ +build/ +RELEASE-VERSION +cbc/tests/output +cbc/tests/data/output diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..466cd00 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +include RELEASE-VERSION diff --git a/cbc/__init__.py b/cbc/__init__.py index 8c167d0..7fc79f7 100644 --- a/cbc/__init__.py +++ b/cbc/__init__.py @@ -1,5 +1,5 @@ from . import environment from . import meta -from . import server from . import utils from . import parsers +from . import cli diff --git a/cbc/cli/__init__.py b/cbc/cli/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cbc/cli/build.py b/cbc/cli/build.py new file mode 100755 index 0000000..ce5fa62 --- /dev/null +++ b/cbc/cli/build.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python +import argparse +import os +import conda_build.metadata +import cbc + + +def main(): + no_upload = '' + use_local = '' + + parser = argparse.ArgumentParser() + parser.add_argument('--force-rebuild', + action='store_true', + help='Do not stop if package already installed') + parser.add_argument('--no-build', + action='store_true', + help='Generate metadata from cbc configuration (useful for manual building)') + parser.add_argument('--no-upload', + action='store_true', + help='Do not upload to anaconda.org (aka. binstar)') + parser.add_argument('--use-local', + action='store_true', + help='Install built package from [...]/conda-bld/pkgs repository') + parser.add_argument('cbcfile', + nargs='+', + help='CBC metadata') + args = parser.parse_args() + + # Initialize cbc internal environment + env = cbc.environment.Environment() + + # Convert cbcfile paths to absolute paths + args.cbcfile = [ os.path.abspath(x) for x in args.cbcfile ] + + # Verify we have a file that exists + for cbcfile in args.cbcfile: + if not os.path.exists(cbcfile): + print('{} does not exist.'.format(cbcfile)) + exit(1) + elif not os.path.isfile(cbcfile): + print('{} is not a file.'.format(cbcfile)) + exit(1) + + if args.no_upload: + no_upload = '--no-binstar-upload' + + if args.use_local: + use_local = '--use-local' + + print('CBC_HOME is {0}'.format(env.cbchome)) + # Perform build(s) + for cbcfile in args.cbcfile: + print('Using cbc build configuration: {0}'.format(cbcfile)) + # Ensure the working directory remains the same throughout. + os.chdir(env.pwd) + + metadata = cbc.meta.MetaData(cbcfile, env) + metadata.env.mkpkgdir(metadata.local['package']['name']) + metadata.render_scripts() + metadata.copy_patches() + + print('Scripts written to {0}'.format(metadata.env.pkgdir)) + + if args.no_build: + continue + + print('Generating Conda metadata...') + conda_metadata = conda_build.metadata.MetaData(env.pkgdir) + + if not args.force_rebuild: + if cbc.utils.conda_search(conda_metadata) == conda_metadata.dist(): + print('{0} matches an installed package; increment the build number to rebuild or use --force-rebuild.'.format(conda_metadata.dist())) + continue + + conda_builder_args = [no_upload, use_local] + try: + print('Initializing Conda build...') + built = cbc.utils.conda_builder(metadata, conda_builder_args) + if not built: + print('Failure occurred building: {0}'.format(conda_metadata.dist())) + continue + except cbc.exceptions.CondaBuildError as cbe: + print(cbe) + continue + + print('Installing Conda package...') + package_exists = cbc.utils.conda_search(conda_metadata) + + if not package_exists: + cbc.utils.conda_install(conda_metadata.name()) + elif package_exists: + if args.force_rebuild: + cbc.utils.conda_reinstall(conda_metadata.name()) + + print('') + + +if __name__ == '__main__': + main() diff --git a/cbc/cli/server.py b/cbc/cli/server.py new file mode 100644 index 0000000..9661f4a --- /dev/null +++ b/cbc/cli/server.py @@ -0,0 +1,53 @@ +import argparse +import os +import http.server +import socketserver +import socket +from threading import Thread + +class Server(socketserver.ThreadingTCPServer): + allow_reuse_address = True + + +class FileServer(object): + def __init__(self, port, root=os.path.abspath(os.curdir), run=False): + if isinstance(port, str): + port = int(port) + + self.root = root + self.port = port + self.handler = http.server.SimpleHTTPRequestHandler + self.httpd = None + + if run: + self.run() + + def run(self, forever=False): + os.chdir(self.root) + socketserver.TCPServer.allow_reuse_address = True + self.httpd = Server(('localhost', self.port), self.handler, True) + print('{0} active on port {1} ({2})'.format(self.__class__.__name__, self.port, self.root)) + if not forever: + self.httpd.handle_request() + else: + self.httpd.serve_forever() + self.close() + + def close(self): + self.httpd.server_close() + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('-r', '--root', default=os.path.abspath(os.curdir), help='Path to files') + parser.add_argument('-p', '--port', type=int, default=8888, help='TCP port') + parser.add_argument('-s', '--single', action='store_false') + args = parser.parse_args() + + fileserver = FileServer(args.port, args.root) + fileserver.run(forever=args.single) + + +if __name__ == '__main__': + main() + diff --git a/cbc/server.py b/cbc/server.py deleted file mode 100644 index 4126ed8..0000000 --- a/cbc/server.py +++ /dev/null @@ -1,53 +0,0 @@ -import argparse -import os -import http.server -import socketserver -import socket -from threading import Thread - -class Server(socketserver.ThreadingTCPServer): - allow_reuse_address = True - - -class FileServer(object): - def __init__(self, port, root=os.path.abspath(os.curdir), run=False): - if isinstance(port, str): - port = int(port) - - self.root = root - self.port = port - self.handler = http.server.SimpleHTTPRequestHandler - self.httpd = None - - if run: - self.run() - - def run(self, forever=False): - os.chdir(self.root) - socketserver.TCPServer.allow_reuse_address = True - self.httpd = Server(('localhost', self.port), self.handler, True) - print('{0} active on port {1} ({2})'.format(self.__class__.__name__, self.port, self.root)) - if not forever: - self.httpd.handle_request() - else: - self.httpd.serve_forever() - self.close() - - def close(self): - self.httpd.server_close() - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('-r', '--root', default=os.path.abspath(os.curdir), help='Path to files') - parser.add_argument('-p', '--port', type=int, default=8888, help='TCP port') - parser.add_argument('-s', '--single', action='store_false') - args = parser.parse_args() - - fileserver = FileServer(args.port, args.root) - fileserver.run(forever=args.single) - - - - - diff --git a/cbc/tests/__init__.py b/cbc/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cbc/tests/data/test.ini b/cbc/tests/data/test.ini new file mode 100644 index 0000000..ec67f6e --- /dev/null +++ b/cbc/tests/data/test.ini @@ -0,0 +1,47 @@ +[cbc_cgi] +local_server: true +local_port: 8888 +local_sources: /srv/conda/sources +protocol: http +url: ${cbc_cgi:protocol}://localhost:${cbc_cgi:local_port} + +[package] +name: test +version: 1.0.0 + +[about] +home: http://example.com/${package:name} +license: GPL +summary: ${package:name} is a test package +readme: README.md + +[source] +fn: ${package:name}-${package:version}.tar.gz +url: ${cbc_cgi:url}/${fn} + +[build] +number: 1 + +[requirements] +build: + python + setuptools + +run: + python + + +[cbc_build] +prefix: /usr/local +win_prefix: c:\anaconda3\ + + +linux: + python setup.py install || exit 1 + +darwin: + python setup.py install || exit 1 + +windows: + python setup.py install + if errorlevel 1 exit 1 \ No newline at end of file diff --git a/cbc/tests/test_cbc.py b/cbc/tests/test_cbc.py new file mode 100644 index 0000000..84e4e67 --- /dev/null +++ b/cbc/tests/test_cbc.py @@ -0,0 +1,66 @@ +import cbc +import nose +import nose.tools +import os +import sys +from cbc.exceptions import IncompleteEnv, MetaDataError + + +class TestCBC(object): + def setUp(self): + lookup = os.path.join(os.path.dirname(__file__), 'data') + output = os.path.join(lookup, 'output') + os.makedirs(output, exist_ok=True) + os.environ['CBC_HOME'] = output + self.env = cbc.environment.Environment() + self.ini = os.path.join(lookup, 'test.ini') + + + def tearDown(self): + pass + + @nose.tools.raises(OSError) + def test_spec_does_not_exist(self): + spec = cbc.meta.MetaData('deadbeefcafe.ini', self.env) + + @nose.tools.raises(IncompleteEnv) + def test_spec_incomplete_environment(self): + '''test_spec_incomplete_environment (a valid ~/.cbcrc will cause this to fail) + ''' + del os.environ['CBC_HOME'] + env = cbc.environment.Environment() + + @nose.tools.raises(MetaDataError) + def test_spec_environment_instance(self): + env = '' + cbc_meta = cbc.meta.MetaData(self.ini, env) + + def test_spec_standalone_build_data(self): + cbc_meta = cbc.meta.MetaData(self.ini, self.env) + nose.tools.assert_in('cbc_build', cbc_meta.local_metadata) + + def test_spec_standalone_cgi_server_data(self): + cbc_meta = cbc.meta.MetaData(self.ini, self.env) + nose.tools.assert_in('cbc_cgi', cbc_meta.local_metadata) + + def test_spec_no_ini_and_yaml_crosstalk(self): + cbc_meta = cbc.meta.MetaData(self.ini, self.env) + nose.tools.assert_not_in('settings', cbc_meta.conda_metadata) + nose.tools.assert_not_in('cbc_build', cbc_meta.conda_metadata) + nose.tools.assert_not_in('cbc_cgi', cbc_meta.conda_metadata) + + def test_spec_outputs_valid_conda_metadata(self): + import conda_build.metadata + cbc_meta = cbc.meta.MetaData(self.ini, self.env) + cbc_meta.env.mkpkgdir(cbc_meta.local['package']['name']) + cbc_meta.render_scripts() + + # Test against conda's build system + conda_meta = conda_build.metadata.MetaData(self.env.pkgdir) + nose.tools.assert_is_instance(conda_meta, conda_build.metadata.MetaData) + nose.tools.assert_equal(conda_meta.dist(), 'test-1.0.0-py34_1') + + + +if __name__ == '__main__': + nose.main(argv=sys.argv) diff --git a/cbcbuild.py b/cbcbuild.py deleted file mode 100755 index 2217a4e..0000000 --- a/cbcbuild.py +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env python -import argparse -import os -import sys -import cbc -import conda_build.metadata - - -#os.environ['CBC_HOME'] = os.path.abspath(os.path.join(os.path.dirname(cbc.__file__), 'tests/data/build')) -#sys.argv.append('--force-rebuild') -#sys.argv.append('tests/data/aprio.ini') - -if __name__ == '__main__': - no_upload = '' - use_local = '' - - parser = argparse.ArgumentParser() - parser.add_argument('--force-rebuild', - action='store_true', - help='Do not stop if package already installed') - parser.add_argument('--no-build', - action='store_true', - help='Generate metadata from cbc configuration (useful for manual building)') - parser.add_argument('--no-upload', - action='store_true', - help='Do not upload to anaconda.org (aka. binstar)') - parser.add_argument('--use-local', - action='store_true', - help='Install built package from [...]/conda-bld/pkgs repository') - parser.add_argument('cbcfile', - nargs='+', - help='CBC metadata') - args = parser.parse_args() - - # Initialize cbc internal environment - env = cbc.environment.Environment() - - # Convert cbcfile paths to absolute paths - args.cbcfile = [ os.path.abspath(x) for x in args.cbcfile ] - - # Verify we have a file that exists - for cbcfile in args.cbcfile: - if not os.path.exists(cbcfile): - print('{} does not exist.'.format(cbcfile)) - exit(1) - elif not os.path.isfile(cbcfile): - print('{} is not a file.'.format(cbcfile)) - exit(1) - - if args.no_upload: - no_upload = '--no-binstar-upload' - - if args.use_local: - use_local = '--use-local' - - print('CBC_HOME is {0}'.format(env.cbchome)) - # Perform build(s) - for cbcfile in args.cbcfile: - print('Using cbc build configuration: {0}'.format(cbcfile)) - # Ensure the working directory remains the same throughout. - os.chdir(env.pwd) - - metadata = cbc.meta.MetaData(cbcfile, env) - metadata.env.mkpkgdir(metadata.local['package']['name']) - metadata.render_scripts() - metadata.copy_patches() - - print('Scripts written to {0}'.format(metadata.env.pkgdir)) - - if args.no_build: - continue - - print('Generating Conda metadata...') - conda_metadata = conda_build.metadata.MetaData(env.pkgdir) - - if not args.force_rebuild: - if cbc.utils.conda_search(conda_metadata) == conda_metadata.dist(): - print('{0} matches an installed package; increment the build number to rebuild or use --force-rebuild.'.format(conda_metadata.dist())) - continue - - conda_builder_args = [no_upload, use_local] - try: - print('Initializing Conda build...') - built = cbc.utils.conda_builder(metadata, conda_builder_args) - if not built: - print('Failure occurred building: {0}'.format(conda_metadata.dist())) - continue - except cbc.exceptions.CondaBuildError as cbe: - print(cbe) - continue - - print('Installing Conda package...') - package_exists = cbc.utils.conda_search(conda_metadata) - - if not package_exists: - cbc.utils.conda_install(conda_metadata.name()) - elif package_exists: - if args.force_rebuild: - cbc.utils.conda_reinstall(conda_metadata.name()) - - print('') diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..ebb0c5c --- /dev/null +++ b/setup.py @@ -0,0 +1,34 @@ +from setuptools import setup, find_packages +from version import get_git_version + +NAME = 'cbc' +VERSION = get_git_version() + +entry_points = {} +package_data = {} + +entry_points['console_scripts'] = [ + 'cbc_build = cbc.cli.build:main', + 'cbc_server = cbc.cli.server:main', +] + +package_data[''] = ['*.txt', '*.md'] +test_suite = 'cbc.tests:main' + +setup( + name=NAME, + version=VERSION, + description='Conda Build Controller', + requires=['conda_build', 'binstar_client'], + provides=[NAME], + author='Joseph Hunkeler', + author_email='jhunk@stsci.edu', + license='BSD', + url='http://bitbucket.org/jhunkeler/cbc', + download_url='', + use_2to3=False, + packages=find_packages(), + entry_points=entry_points, + package_data=package_data, + test_suite=test_suite, +) diff --git a/tests/data/test.ini b/tests/data/test.ini deleted file mode 100644 index ec67f6e..0000000 --- a/tests/data/test.ini +++ /dev/null @@ -1,47 +0,0 @@ -[cbc_cgi] -local_server: true -local_port: 8888 -local_sources: /srv/conda/sources -protocol: http -url: ${cbc_cgi:protocol}://localhost:${cbc_cgi:local_port} - -[package] -name: test -version: 1.0.0 - -[about] -home: http://example.com/${package:name} -license: GPL -summary: ${package:name} is a test package -readme: README.md - -[source] -fn: ${package:name}-${package:version}.tar.gz -url: ${cbc_cgi:url}/${fn} - -[build] -number: 1 - -[requirements] -build: - python - setuptools - -run: - python - - -[cbc_build] -prefix: /usr/local -win_prefix: c:\anaconda3\ - - -linux: - python setup.py install || exit 1 - -darwin: - python setup.py install || exit 1 - -windows: - python setup.py install - if errorlevel 1 exit 1 \ No newline at end of file diff --git a/tests/test.py b/tests/test.py deleted file mode 100644 index 49e22a5..0000000 --- a/tests/test.py +++ /dev/null @@ -1,70 +0,0 @@ -import nose -import nose.tools -import os -import cbc -from cbc.exceptions import IncompleteEnv, MetaDataError -import sys - - -class TestCBC(object): - def setUp(self): - lookup = os.path.join(os.path.dirname(__file__), 'data') - output = os.path.join(lookup, 'output') - os.makedirs(output, exist_ok=True) - os.environ['CBC_HOME'] = output - self.env = cbc.environment.Environment() - self.ini = os.path.join(lookup, 'test.ini') - - - def tearDown(self): - pass - - @nose.tools.raises(OSError) - def test_spec_does_not_exist(self): - '''Issue non-existent INI and see what happens. - ''' - spec = cbc.meta.MetaData('deadbeefcafe.ini', self.env) - - @nose.tools.raises(IncompleteEnv) - def test_spec_incomplete_environment(self): - '''Screw up the environment on purpose - ''' - del os.environ['CBC_HOME'] - env = cbc.environment.Environment() - - @nose.tools.raises(MetaDataError) - def test_spec_environment_instance(self): - '''Issue the incorrect class instance as the environment - ''' - env = '' - cbc_meta = cbc.meta.MetaData(self.ini, env) - - def test_spec_standalone_build_data(self): - cbc_meta = cbc.meta.MetaData(self.ini, self.env) - nose.tools.assert_in('cbc_build', cbc_meta.local_metadata) - - def test_spec_standalone_cgi_server_data(self): - cbc_meta = cbc.meta.MetaData(self.ini, self.env) - nose.tools.assert_in('cbc_cgi', cbc_meta.local_metadata) - - def test_spec_no_ini_and_yaml_crosstalk(self): - cbc_meta = cbc.meta.MetaData(self.ini, self.env) - nose.tools.assert_not_in('cbc_build', cbc_meta.conda_metadata) - nose.tools.assert_not_in('cbc_cgi', cbc_meta.conda_metadata) - - def test_spec_outputs_valid_conda_metadata(self): - import conda_build.metadata - cbc_meta = cbc.meta.MetaData(self.ini, self.env) - cbc_meta.env.mkpkgdir(cbc_meta.local['package']['name']) - cbc_meta.render_scripts() - - # Test against conda's build system - conda_meta = conda_build.metadata.MetaData(self.env.pkgdir) - nose.tools.assert_is_instance(conda_meta, conda_build.metadata.MetaData) - nose.tools.assert_equal(conda_meta.dist(), 'test-1.0.0-py34_1') - - - -if __name__ == '__main__': - sys.argv.append('--verbosity=3') - nose.main(argv=sys.argv) diff --git a/version.py b/version.py new file mode 100644 index 0000000..ae37850 --- /dev/null +++ b/version.py @@ -0,0 +1,104 @@ +# -*- coding: utf-8 -*- +# Author: Douglas Creager +# This file is placed into the public domain. + +# Calculates the current version number. If possible, this is the +# output of “git describe”, modified to conform to the versioning +# scheme that setuptools uses. If “git describe” returns an error +# (most likely because we're in an unpacked copy of a release tarball, +# rather than in a git working copy), then we fall back on reading the +# contents of the RELEASE-VERSION file. +# +# To use this script, simply import it your setup.py file, and use the +# results of get_git_version() as your package version: +# +# from version import * +# +# setup( +# version=get_git_version(), +# . +# . +# . +# ) +# +# This will automatically update the RELEASE-VERSION file, if +# necessary. Note that the RELEASE-VERSION file should *not* be +# checked into git; please add it to your top-level .gitignore file. +# +# You'll probably want to distribute the RELEASE-VERSION file in your +# sdist tarballs; to do this, just create a MANIFEST.in file that +# contains the following line: +# +# include RELEASE-VERSION +from __future__ import print_function +from subprocess import Popen, PIPE + +__all__ = ("get_git_version") + + +def call_git_describe(abbrev=4): + try: + p = Popen(['git', 'describe', '--abbrev=%d' % abbrev], + stdout=PIPE, stderr=PIPE) + p.stderr.close() + line = p.stdout.readlines()[0] + return line.strip() + + except: + return None + + +def read_release_version(): + try: + f = open("RELEASE-VERSION", "r") + + try: + version = f.readlines()[0] + return version.strip() + + finally: + f.close() + + except: + return None + + +def write_release_version(version): + f = open("RELEASE-VERSION", "w") + f.write("%s\n" % version) + f.close() + + +def get_git_version(abbrev=4): + # Read in the version that's currently in RELEASE-VERSION. + + release_version = read_release_version() + + # First try to get the current version using “git describe”. + + version = call_git_describe(abbrev) + + # If that doesn't work, fall back on the value that's in + # RELEASE-VERSION. + + if version is None: + version = release_version + + # If we still don't have anything, that's an error. + + if version is None: + raise ValueError("Cannot find the version number!") + + # If the current version is different from what's in the + # RELEASE-VERSION file, update the file to be current. + + if version != release_version: + write_release_version(version) + + # Finally, return the current version. + + return version + + +if __name__ == "__main__": + print(get_git_version()) -- cgit