Turning into a package

3 files changed, 494 insertions, 0 deletions

diff --git a/rambo/__init__.py b/rambo/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/rambo/__init__.py
diff --git a/rambo/_version.py b/rambo/_version.py
new file mode 100644
index 0000000..a655b17
--- /dev/null
+++ b/rambo/_version.py
@@ -0,0 +1 @@
+__version__ = '1.0.0b1'
diff --git a/rambo/rambo.py b/rambo/rambo.py
new file mode 100755
index 0000000..998c2f8
--- /dev/null
+++ b/rambo/rambo.py
@@ -0,0 +1,493 @@
+#!/usr/bin/env python
+
+'''
+RAMBO - Recipe Analyzer and Multi-package Build Optimizer
+
+Requires conda & conda-build to be installed in a path that appears in the
+python interprer's search list in order to access the API machinery via
+'conda_build.api.
+'''
+
+from __future__ import print_function
+import os
+import sys
+from copy import deepcopy
+import argparse
+from six.moves import urllib
+import codecs
+from yaml import safe_load
+import json
+from rambo._version import __version__
+try:
+    import conda_build.api
+except ImportError:
+    raise ImportError('conda-build must be installed order to use this \n'
+                      'tool. Either conda-build is not installed, or you \n'
+                      'are working in an activated conda environment. \n'
+                      'If conda-build is installed deactivate the \n'
+                      'environment currently enabled or explicitly switch \n'
+                      'to the conda "root" environment to allow use of\n'
+                      'conda-build.')
+
+DEFAULT_MINIMUM_NUMPY_VERSION = '1.11'
+
+
+class meta(object):
+    '''Holds metadata for a recipe obtained from the recipe's meta.yaml file,
+    certain values derived from that data, and methods to calculate those
+    derived values.'''
+
+    def __init__(self, recipe_dir, versions, dirty=False):
+        self.recipe_dirname = os.path.basename(recipe_dir)
+        self.versions = versions
+        self.dirty = dirty
+        self.metaobj = None     # renderdata[0] (MetaData)
+        self.mdata = None       # renderdata[0].meta (dict)
+        self.active = True  # Visit metadata in certain processing steps?
+        self.valid = False
+        self.complete = False
+        self.name = None
+        self.num_bdeps = 0
+        self.deps = []
+        self.peer_bdeps = []
+        self.import_metadata(recipe_dir)
+        self.derive_values()
+        self.canonical_name = ''
+        # Whether or not the package with this metadata
+        # already exists in the channel archive
+        self.archived = False
+        self.gen_canonical()
+
+    def import_metadata(self, rdir):
+        '''Read in the package metadata from the given recipe directory via
+        the conda recipe renderer to perform string interpolation and
+        store the values in a dictionary.'''
+        if os.path.isfile(rdir + '/meta.yaml'):
+            # render() returns a tuple: (MetaData, bool, bool)
+            self.metaobj = conda_build.api.render(
+                rdir,
+                dirty=self.dirty,
+                python=self.versions['python'],
+                numpy=self.versions['numpy'])[0]
+            self.mdata = self.metaobj.meta
+            self.valid = self.is_valid()
+            self.complete = self.is_complete()
+            if self.valid:
+                self.name = self.mdata['package']['name']
+        else:
+            print('Recipe directory {0} has no meta.yaml file.'.format(
+                self.recipe_dirname))
+
+    def derive_values(self):
+        if self.complete:
+            self.num_bdeps = len(self.mdata['requirements']['build'])
+            for req in self.mdata['requirements']['build']:
+                self.deps.append(req.split()[0])
+
+    def deplist(self, deptype):
+        '''Return the simplified (no version info, if present) list of
+        dependency names of the given type.'''
+        lst = []
+        for dep in self.mdata['requirements'][deptype]:
+            lst.append(dep.split()[0])
+        return lst
+
+    def is_valid(self):
+        '''Does the metadata for this recipe contain the minimum information
+        necessary to process?'''
+        valid = True
+        if 'package' not in self.mdata.keys():
+            complete = False
+        return valid
+
+    def is_complete(self):
+        '''Is the metadata for this recipe complete enough to allow for use
+        in build-order optimization?'''
+        complete = True
+        if 'requirements' in self.mdata.keys():
+            if 'build' not in self.mdata['requirements'].keys():
+                complete = False
+        else:
+            complete = False
+        return complete
+
+    def gen_canonical(self):
+        '''Generate the package's canonical name using available
+        information.'''
+        self.canonical_name = os.path.basename(
+                conda_build.api.get_output_file_path(
+                    self.metaobj,
+                    python=self.versions['python'],
+                    numpy=self.versions['numpy']))
+
+
+class metaSet(object):
+    '''A collection of mulitple recipe metadata objects from a directory
+    specification, and methods for manipulationg and querying this
+    collection.'''
+
+    ignore_dirs = ['.git', 'template']
+
+    def __init__(self,
+                 directory,
+                 versions,
+                 platform,
+                 manfile=None,
+                 dirty=False):
+        '''Parameters:
+        directory - a relative or absolute directory in which Conda
+          recipe subdirectories may be found.
+        versions - Dictionary containing python, numpy, etc, version
+          information.'''
+        self.metas = []
+        self.platform = platform
+        self.versions = versions
+        self.manfile = manfile
+        self.manifest = None
+        if self.manfile:
+            self.read_manifest()
+            self.filter_by_manifest()
+        self.dirty = dirty
+        self.incomplete_metas = []
+        self.names = []
+        self.read_recipes(directory)
+        self.derive_values()
+        self.sort_by_peer_bdeps()
+        self.merge_metas()
+        if self.channel:
+            self.channel_data = self.get_channel_data()
+            self.flag_archived()
+
+    def read_recipes_old(self, directory):
+        '''Process a directory reading in each conda recipe found, creating
+        a list of metadata objects for use in analyzing the collection of
+        recipes as a whole.'''
+        recipe_dirnames = os.listdir(directory)
+        for rdirname in recipe_dirnames:
+            if rdirname in self.ignore_dirs:
+                continue
+            rdir = directory + '/' + rdirname
+            m = meta(rdir, versions=self.versions, dirty=self.dirty)
+            if m.complete:
+                self.metas.append(m)
+                self.names.append(m.name)
+            else:
+                self.incomplete_metas.append(m)
+
+    def read_recipe_selection(self, directory, recipe_list):
+        '''Process a directory reading in each conda recipe found, creating
+        a list of metadata objects for use in analyzing the collection of
+        recipes as a whole.'''
+        for rdirname in recipe_list:
+            if rdirname in self.ignore_dirs:
+                continue
+            rdir = directory + '/' + rdirname
+            m = meta(rdir, versions=self.versions, dirty=self.dirty)
+            if m.complete:
+                self.metas.append(m)
+                self.names.append(m.name)
+            else:
+                self.incomplete_metas.append(m)
+
+    def read_recipes(self, directory):
+        recipe_dirnames = os.listdir(directory)
+        # If a manifest was given, use it to filter the list of available
+        # recipes.
+        if self.manifest:
+            recipe_list = set.intersection(
+                set(recipe_dirnames),
+                set(self.manifest['packages']))
+        else:
+            recipe_list = recipe_dirnames
+        self.read_recipe_selection(directory, recipe_list)
+
+    def read_manifest(self):
+        mf = open(self.manfile, 'r')
+        self.manifest = safe_load(mf)
+        self.channel = self.manifest['channel_URL'].strip('/')
+        self.channel += '/' + self.platform
+        self.versions['numpy'] = str(self.manifest['numpy_version'])
+
+    def filter_by_manifest(self):
+        '''Leave only the recipe metadata entries that appear in the
+        provided manifest list active.'''
+        for meta in self.metas:
+            if meta.name not in self.manifest['packages']:
+                meta.active = False
+
+    def merge_metas(self):
+        '''Prepend the list of metas that do not have complete build
+        dependency information to the main list.
+        Also, add those names to the names list.'''
+        # Sort alphabetically by name
+        self.incomplete_metas = sorted(
+            self.incomplete_metas,
+            key=lambda meta: meta.name)
+        for m in self.incomplete_metas[::-1]:
+            self.metas.insert(0, m)
+
+    def derive_values(self):
+        '''Produce values from the set of recipes taken as a whole.'''
+        self.calc_peer_bdeps()
+
+    def calc_peer_bdeps(self):
+        '''Produce and store a names-only list of the build dependencies
+        for each recipe found to this set of recipes that each recipe
+        references.'''
+        for meta in self.metas:
+            for name in meta.deps:
+                if name in self.names:
+                    meta.peer_bdeps.append(name)
+
+    def sort_by_peer_bdeps(self):
+        '''Sort the list of metadata objects by the number of peer build
+        dependencies each has, in ascending order. This gives a good first
+        approximation to a correct build order of all peers.  Peform an
+        extra step here to reduce stochasticity of the order of packages
+        within a given tier that all share the same number of peer_bdeps.
+        The order of those items apparently varies from run to run.'''
+        # First sort by alphabetical on name to make the subsequent
+        # sorting deterministic.
+        self.metas = sorted(self.metas, key=lambda meta: meta.name)
+        self.metas = sorted(self.metas, key=lambda meta: len(meta.peer_bdeps))
+
+    def index(self, mname):
+        '''Return the index of a metadata object with the name 'mname'.'''
+        for i, meta in enumerate(self.metas):
+            if (meta.name == mname):
+                return i
+        raise IndexError('Name [{0}] not found.'.format(mname))
+
+    def peer_bdep_indices(self, mname):
+        '''Returns a list of the indices in the meta list corresponding to
+        all the peer build dependencies (bdeps) of the given package
+        metadata.'''
+        indices = []
+        for i, meta in enumerate(self.metas):
+            if (meta.name == mname):
+                for dep in meta.peer_bdeps:
+                    indices.append(self.index(dep))
+        return indices
+
+    def position_OK(self, mname):
+        '''If a package has peer build dependencies that all occur before
+        the package in the sorted list of package recipes, the package's
+        position in the build order list is acceptable.'''
+        for i in self.peer_bdep_indices(mname):
+            if i > self.index(mname):
+                return False
+        return True
+
+    def relocate(self, mname):
+        '''Relocate a meta object in the meta set such that all its internal
+        dependencies appear earlier in the list than it does.
+        The algorithm:
+        For a package that does not have position_OK=True, examine the
+        internal dependency indices. If any index is greater than the
+        package's index, relocate the package to the index in the list just
+        after the largest such dependency index.
+        1. Deepcopy object into temp variable
+        2. Insert copy into list at new index
+        3. remove the original item from list'''
+        idx = self.index(mname)
+        new_idx = max(self.peer_bdep_indices(mname)) + 1
+        temp = deepcopy(self.metas[idx])
+        self.metas.insert(new_idx, temp)
+        del self.metas[idx]
+
+    def optimize_build_order(self):
+        '''Makes a single pass through the list of (complete) package metadata,
+        relocating in the list any item which is not in the correct slot in
+        the build order.'''
+        for m in self.metas:
+            if not self.position_OK(m.name):
+                self.relocate(m.name)
+
+    def multipass_optimize(self, max_passes=8):
+        '''Makes multiple passes over the list of metadata, optimizing during
+        each pass until either the entire list is ordered correctly for
+        building, or the maximum number of allowed passes is reached. The
+        latter condition suggests there is a circular dependency that needs
+        to be manually resolved.'''
+        opass = 0
+        num_notOK = 1
+        while (num_notOK > 0 and opass < max_passes):
+            opass = opass + 1
+            num_notOK = 0
+            self.optimize_build_order()
+            for m in self.metas:
+                if not self.position_OK(m.name):
+                    num_notOK = num_notOK + 1
+        if (opass == max_passes):
+            print('Pass {0} of {1} reached. Check for circular '
+                  'dependencies.'.format(
+                    opass,
+                    max_passes))
+            return False
+        return True
+
+    def get_channel_data(self):
+        '''Download the channel metadata from all specified conda package
+        channel URLs, parse the JSON data into a dictionary.'''
+        jsonbytes = urllib.request.urlopen(self.channel + '/repodata.json')
+        # urllib only returns 'bytes' objects, so convert to unicode.
+        reader = codecs.getreader('utf-8')
+        return json.load(reader(jsonbytes))
+
+    def flag_archived(self):
+        '''Flag each meta as either being archived or not by generating the
+        package canonical name, fetching the provided conda channel
+        archive data, and searching the archive data for the generated
+        name. Each meta's 'archived' attribute is set to True if found
+        and False if not.'''
+        for meta in self.metas:
+            if meta.canonical_name in self.channel_data['packages'].keys():
+                meta.archived = True
+
+    def print_details(self, fh=sys.stdout):
+        num_notOK = 0
+        print('conda-build version     : ', conda_build.__version__)
+        print('Python version specified: ', self.versions['python'])
+        print('Numpy  version specified: ', self.versions['numpy'])
+        print('                              num  num      peer', file=fh)
+        print('         name               bdeps  peer     bdep     pos.',
+              file=fh)
+        print('                                   bdeps    indices  OK?',
+              file=fh)
+        print('----------------------------------------------------------',
+              file=fh)
+        for idx, m in enumerate(self.metas):
+            if not self.position_OK(m.name):
+                num_notOK = num_notOK + 1
+            print('{0:>28}  {1:{wid}}  {2:{wid}}  idx={3:{wid}} {4} {5}'
+                  .format(m.name,
+                          m.num_bdeps,
+                          len(m.peer_bdeps),
+                          idx,
+                          self.peer_bdep_indices(m.name),
+                          self.position_OK(m.name),
+                          wid=2), file=fh)
+        print('Num not in order = {0}/{1}\n'.format(
+            num_notOK,
+            len(self.metas)), file=fh)
+
+    def print(self, fh=sys.stdout):
+        '''Prints the list of package names in the order in which they appear
+        in self.metas to stdout, suitable for ingestion by other tools during
+        a build process.'''
+        for m in self.metas:
+            print('{0}'.format(m.name), file=fh)
+
+    def print_culled(self, fh=sys.stdout):
+        '''Prints the list of package names for which the canonical name does
+        not exist in the specified archive channel. List is presented in the
+        order in which entries appear in self.metas.'''
+        for m in [m for m in self.metas if m.active and not m.archived]:
+            print('{0}'.format(m.name), file=fh)
+
+    def print_canonical(self, fh=sys.stdout):
+        '''Prints list of canonical package names.'''
+        for meta in self.metas:
+            print('{0:>50}'.format(meta.canonical_name), file=fh)
+
+    def print_status_in_channel(self, fh=sys.stdout):
+        '''Prints list of canonical package names and whether or not each
+        has already been built and archived in the specified channel.'''
+        statstr = {True: '', False: 'Not in channel archive'}
+        for meta in self.metas:
+            print('{0:>50}   {1}'.format(
+                meta.canonical_name,
+                statstr[meta.archived]), file=fh)
+
+
+# ----
+
+
+def main(argv=None):
+
+    if argv is None:
+        argv = sys.argv
+
+    parser = argparse.ArgumentParser(prog='rambo')
+    parser.add_argument('-p', '--platform', type=str)
+    parser.add_argument(
+            '--python',
+            type=str,
+            help='Python version to pass to conda machinery when rendering '
+            'recipes. "#.#" format. If not specified, the version of python'
+            ' hosting conda_build.api is used.')
+    parser.add_argument(
+            '-m',
+            '--manifest',
+            type=str,
+            help='Use this file to filter the list of recipes to process.')
+    parser.add_argument(
+            '-f',
+            '--file',
+            type=str,
+            help='Send package list output to this file instead of stdout.')
+    parser.add_argument(
+            '-c',
+            '--culled',
+            action='store_true',
+            help='Print the ordered list of package names reduced to the set'
+            ' of packages that do not already exist in the channel specified'
+            ' in the supplied manifest file.')
+    parser.add_argument(
+            '-d',
+            '--details',
+            action='store_true',
+            help='Display details used in determining build order and/or '
+            'package culling.')
+    parser.add_argument(
+            '--dirty',
+            action='store_true',
+            help='Use the most recent pre-existing conda work directory for '
+            'each recipe instead of creating a new one. If a work directory '
+            'does not already exist, the recipe is processed in the normal '
+            'fashion. Used mostly for testing purposes.')
+    parser.add_argument(
+            '-v',
+            '--version',
+            action='version',
+            version='%(prog)s ' + __version__,
+            help='Display version information.')
+    parser.add_argument('recipes_dir', type=str)
+    args = parser.parse_args()
+
+    if args.version:
+        print(__version__)
+        os.exit(0)
+
+    recipes_dir = os.path.normpath(args.recipes_dir)
+
+    fh = None
+    if args.file:
+        fh = open(args.file, 'w')
+
+    versions = {'python': '', 'numpy': ''}
+    if args.python:
+        versions['python'] = args.python
+
+    versions['numpy'] = DEFAULT_MINIMUM_NUMPY_VERSION
+
+    mset = metaSet(
+            recipes_dir,
+            platform=args.platform,
+            versions=versions,
+            dirty=args.dirty,
+            manfile=args.manifest)
+
+    mset.multipass_optimize()
+
+    if args.details:
+        mset.print_details(fh)
+        if mset.channel:
+            mset.print_status_in_channel(fh)
+    elif args.culled:
+        mset.print_culled(fh)
+    else:
+        mset.print(fh)
+
+if __name__ == "__main__":
+    main()