Create release_notes and package_manifest automatically (#24)

* auto creating release_notes and package_manifest

release_notes now provide links to individual
repository release notes instead of collecting them
together

Using anonymous access with github3.  Note that this
has API call restrictions, so can only do so many times
within an hour before being blocked.

setup.py install now runs the two scripts in source
to generate the manifest and release notes rst files.

minor changes to py scripts to use different org access
calls, and defining output path to not be CWD.

requirement and calls to pandoc have also been removed.
files are written directly to rst instead of being
written to md and converted.

* Exclude non-astroconda packages from release_notes

* change disclaimer at top of release notes

1 files changed, 71 insertions, 132 deletions

diff --git a/source/release_notes.py b/source/release_notes.py
index 3c1b397..fc3bbb3 100644
--- a/source/release_notes.py
+++ b/source/release_notes.py
@@ -6,176 +6,115 @@ import math
 from collections import OrderedDict
 
 try:
-    from github3 import login
+    from urllib.request import urlopen
+except ImportError:
+    from urllib2 import urlopen
+
+try:
+    import github3
+    from github3 import organization
     from github3 import GitHubError
 except ImportError:
     print('github3.py required.')
     print('pip install https+git://github.com/sigmavirus24/github3.py')
     exit(1)
 
-try:
-    import pypandoc
-except ImportError:
-    print('pypandoc required.')
-    print('conda install -c https://conda.anaconda.org/janschulz pypandoc')
-    exit(1)
+from package_manifest import ARCHITECTURE, get_repodata
 
 
 header = """
-# Release Notes
+Release Notes
+=============
+
+For individual package release notes, please follow the links below.
+Note that not all packages have release notes, and may simply provide
+numbered releases.
+
 """
-title = """
-## {name}
+
+item = """
+-  `{} <{}>`__
 """
-message = """
-### {title}
 
-*{date}*
+#-------------------------------------------------------------------------------
 
-{body}
+def any_releases(url):
+    """Check if there are any releases in a given github release page
 
-"""
+    Parameters
+    ----------
+    url: str
+        URL of a valid github reposoitory release pages
 
+    Returns
+    -------
+    any_releases: bool
+        True if there are found releases, False otherwise
 
-def explode_newlines(s):
-    """Normalize newlines and double them up for MD->RST conversion.
-    """
-    tmp = ''
-    for line in s.splitlines():
-        line = line.replace('\r\n', '\n')
-        line = line + '\n\n'
-        tmp += line
-
-    return tmp
-
-def transform_headings(s):
-    """Trust no one.
-    Header levels less than or equal to 3 are up-converted.
-
-    1 = 4
-    2 = 4
-    3 = 5
-    4 = 6
-    5 = 6
-    6 = 6
-
-    Oh well, sucks right?
-    """
-    delim = '#'
-    headings = OrderedDict(
-        h6=delim * 6,
-        h5=delim * 5,
-        h4=delim * 4,
-        h3=delim * 3,
-        h2=delim * 2,
-        h1=delim * 1
-    )
-    output = ''
-
-    for line in s.splitlines():
-        length = len(line)
-        stop = len(headings.keys()) + 1
-
-        if length < stop:
-            stop = length
-
-        block = line[:stop]
-        depth = 0
-        for i, ch in enumerate(block, 1):
-            if ch != '#': break
-            depth = i
-
-        if depth == 0 or depth > 3:
-            output += explode_newlines(line)
-            continue
-
-        #print('length={:<10d} stop={:<10d} depth={:<10d} block={:>20s}'.format(length, stop, depth, block))
-
-        current = headings['h'+str(depth)]
-        try:
-            required = headings['h'+str(math.ceil(depth + depth / depth + 1))]
-        except KeyError:
-            required = headings['h'+str(len(headings) - 1)]
-
-        if depth > 0 and depth <= 3:
-            print("Heading levels 1, 2, and 3 are allocated by this script...")
-            print("Offending line: '{0}'".format(line))
-            print("Automatically converted to: '{0}'".format(required))
-            print("(FIX YOUR RELEASE NOTES)\n\n")
-
-        #print('Transformed "{0}" -> "{1}"'.format(current, required))
-        line = line.replace(current, required)
-        output += explode_newlines(line)
-
-    return output
-
-def read_token(filename):
-    """For the sake of security, paste your github auth token in a file and reference it using this function
-
-    Expected file format (one-liner):
-    GITHUB_USER_AUTH_TOKEN_HERE
     """
-    with open(filename, 'r') as f:
-        # Obtain the first line in the token file
-        token = f.readline().rstrip()
 
-    if not token:
-        return None
+    no_release_msg = b"<h3>There aren\xe2\x80\x99t any releases here</h3>"
 
-    for i, ch in enumerate(token):
-        if not ch.isalnum():
-            raise ValueError('Invalid token: Non-alphanumeric character detected (Value: "{0}", ASCII: {1}, Index: {2}) '.format(ch, ord(ch), i))
+    if no_release_msg in urlopen(url).read():
+        return False
 
-    return token
+    return True
 
+#-------------------------------------------------------------------------------
 
-def pull_release_notes(tmpfile, outfile):
-    with open(tmpfile, 'w+') as mdout:
+def pull_release_notes(org, outfile):
+    """ Get urls for release notes for all astroconda packages
+
+    Only repositories that are included in ANY astroconda distribution (linux or OSX)
+    with tagged github releases will be included in the output release notes
+    rst file.
+
+    Parameters
+    ----------
+    org: github3 organization instance
+        All repositories from this org will be parsed for releases
+    outfile: str
+        Filename to write urls to.
+
+    """
+
+    astroconda_packages = {item['name'] for arch in ARCHITECTURE for item in get_repodata(arch).values()}
+
+    with open(outfile, 'w+') as mdout:
         # Write header (main title)
         print(header, file=mdout)
 
         # Sort repositories inline alphabetically, because otherwise its seemingly random order
-        for repository in sorted(list(org.repositories()), key=lambda k: k.as_dict()['name']):
-            repo = repository.as_dict()
-            repo_name = repo['name'].upper()
+        for repository in sorted(list(org.iter_repos()), key=lambda k: k.name):
 
-            # Ignore repositories without release notes
-            releases = list(repository.releases())
-            if not releases:
+            if not repository.name in astroconda_packages:
                 continue
 
-            # Write repository title
-            print(title.format(name=repo_name), file=mdout)
+            release_url = repository.html_url + '/releases'
 
-            for note in repository.releases():
-                # RST is EXTREMELY particular about newlines.
-                body = explode_newlines(note.body)
-                body = transform_headings(note.body)
-                # Write release notes structure
-                print(message.format(title=note.name,
-                                 date=note.published_at,
-                                 body=body), file=mdout)
+            if not any_releases(release_url):
+                continue
 
+            print(item.format(repository.name.upper(), release_url), file=mdout)
+
+#-------------------------------------------------------------------------------
+
+def generate_release_notes():
+    """Main function to create the release_notes.rst pages
+    """
 
-if __name__ == '__main__':
     owner = 'spacetelescope'
-    tmpfile = 'release_notes.md'
-    outfile = 'release_notes.rst'
+    org = github3.organization(owner)
 
-    gh = login(token=read_token('TOKEN'))
-    org = gh.organization(owner)
+    outfile = os.path.join('source', 'release_notes.rst')
 
     try:
-        pull_release_notes(tmpfile, outfile)
+        pull_release_notes(org, outfile)
     except GitHubError as e:
         print(e)
         exit(1)
 
-    if os.path.exists(outfile):
-        os.unlink(outfile)
+#-------------------------------------------------------------------------------
 
-    if os.path.exists(tmpfile):
-        pypandoc.convert(source=tmpfile,
-                         to='rst',
-                         outputfile=outfile)
-        os.unlink(tmpfile)
+if __name__ == '__main__':
+    generate_release_notes()