diff options
| -rw-r--r-- | .gitignore | 12 | ||||
| -rw-r--r-- | LICENSE.txt | 30 | ||||
| -rw-r--r-- | MANIFEST.in | 2 | ||||
| -rw-r--r-- | README.md | 69 | ||||
| -rw-r--r-- | firewatch/__init__.py | 0 | ||||
| -rw-r--r-- | firewatch/firewatch.py | 239 | ||||
| -rw-r--r-- | setup.py | 57 |
7 files changed, 409 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3b302c1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +RELIC-INFO +relic/ +*/version.py + +build/ +dist/ +*.pyc +*.o +*.so +*.egg-info +*.swp +*.json* diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..a0b349a --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,30 @@ +BSD 3-Clause License + +Copyright (c) 2018, Space Telescope Science Institute, AURA +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..423c68c --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +include RELIC-INFO +recursive-include firewatch * diff --git a/README.md b/README.md new file mode 100644 index 0000000..a547c76 --- /dev/null +++ b/README.md @@ -0,0 +1,69 @@ +# Firewatch + +Is your user-base complaining their environment(s) broke after running `conda update [your_package]`? I bet they are! Have you ever wondered what packages were introduced into the Conda ecosystem without your knowledge? Wonder no more! + +## License + +BSD 3-clause + +## Support + +- None. +- Use at your own risk. + +## Requirements + +This utility requires Python 3.6 or greater and `requests`. + +## Usage + +``` +$ firewatch --help +usage: firewatch [-h] [--benchmark] [--brute-force] [--channel CHANNELS] + [--order ORDER] [--platform PLATFORMS] + [--time-span TIME_SPAN] + +optional arguments: + -h, --help show this help message and exit + --benchmark Display total time to parse and sort channel data + --brute-force Derive timestamps from HTTP header: "last-modified" + --channel CHANNELS, -c CHANNELS + Conda channel + --order ORDER, -o ORDER + [asc|dsc] + --platform PLATFORMS, -p PLATFORMS + [linux|osx|win]-[32|64] + --time-span TIME_SPAN, -t TIME_SPAN + i[s|m|h|d|w|M|y|D|c] (120s, 12h, 1d, 2w, 3m, 4y) +``` + + +## Examples + +_What's probably broken within..._ + +... the last 24 hours? + +```bash +$ firewatch -t 1d +``` + +... the last week? + +```bash +$ firewatch -t 1w +``` + +... three days on some high profile anaconda.org channels? + +```bash +$ firewatch -t 3d -c conda-forge -c intel +``` + +... the last month on AstroConda? + +```bash +$ firewatch -t 1M -c http://ssb.stsci.edu/astroconda +``` + + diff --git a/firewatch/__init__.py b/firewatch/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/firewatch/__init__.py diff --git a/firewatch/firewatch.py b/firewatch/firewatch.py new file mode 100644 index 0000000..df17e8a --- /dev/null +++ b/firewatch/firewatch.py @@ -0,0 +1,239 @@ +import errno +import json +import platform as PLATFORM +import requests +import sys +import time +from datetime import datetime, timedelta + + +conda_channel_pool = [ + f'https://repo.continuum.io/pkgs/main', + # f'https://repo.continuum.io/pkgs/free', # deprecated: conda<4.3.30 +] + +time_units = dict( + s=1, # second + m=60, # minute + h=3600, # hour + d=86400, # day + w=604800, # week + M=2.628e+6, # month + y=3.154e+7, # year + D=3.154e+8, # decade + c=3.154e+9, # century +) + +system_map = dict( + Linux='linux', + Darwin='osx', + Windows='win' +) + +machine_map = dict( + i386='32', + x86_64='64' +) + + +def extract_channel_platform(url): + """Returns last two elements in URL: (channel/platform-arch) + """ + parts = [x for x in url.split('/')] + result = '/'.join(parts[-2:]) + return result + + +def convert_human_timespan(t): + """Convert timespan to seconds to generate datetime.timedelta objects + """ + value, unit = int(t[:-1]), t[-1] + if unit not in time_units.keys(): + raise ValueError(f'Invalid time unit: "{unit}" (expected: [' + f'{"|".join([x for x in time_units.keys()])}])') + return value * time_units[unit] + + +def get_packages(channels): + packages = list() + for channel in channels: + repodata = f'{channel}/repodata.json' + data = dict( + packages=list(), + channel=extract_channel_platform(channel), + ) + + try: + with requests.get(repodata) as r: + r.raise_for_status() + data['packages'] = json.loads(r.text)['packages'] + packages.append(data) + + except requests.exceptions.RequestException as e: + print(f'Error {e.response.status_code}/{e.response.reason}:' + f' {channel}', file=sys.stderr) + except Exception as e: + print(e) + + return packages + + +def get_timestamps(data, brute_force=False): + """ Extract and convert package timestamps to datetime objects + """ + rt_fmt = '%a, %d %b %Y %H:%M:%S %Z' + + for base in data: + for pkg_name, pkg_info in base['packages'].items(): + result = dict() + result['name'] = pkg_name + result['channel'] = base['channel'] + + timestamp = datetime(1970, 1, 1) + # Continuum used 'date' for tracking some time ago + if 'date' in pkg_info: + date_str = [int(x) for x in pkg_info['date'].split('-')] + timestamp = datetime(*date_str) + + # Newer packages use 'timestamp', but depending on the direction + # of the wind, the unix epoch is stored in microseconds rather + # than seconds. So adjust for former case... + elif 'timestamp' in pkg_info: + timestamp = datetime.fromtimestamp(pkg_info['timestamp'] // 1000) + if timestamp < datetime(2000, 1, 1): + timestamp = datetime.fromtimestamp(pkg_info['timestamp']) + + # Scan remote server for 'last-modified' timestamp + # Don't do this unless you own the server you're spamming. + elif brute_force: + url = f'{result["channel"]}/{pkg_name}' + try: + modified = requests.head(url).headers['last-modified'] + except requests.exceptions.RequestException as e: + print(f'Error {e.response.status_code}/{e.response.reason}:' + f' {result["channel"]}', file=sys.stderr) + continue + except Exception as e: + print(e) + continue + + timestamp = datetime.strptime(modified, rt_fmt) + + result['timestamp'] = timestamp + yield result + + +def noarch_channel(channel, platform): + channel = channel.replace(f'{platform}', 'noarch') + return channel + + +def convert_channel(channel, platform, noarch=False): + # Strip trailing slash + if channel.endswith('/'): + channel = channel[:-1] + + # Sanitize URL by stripping out part we will adjust dynamically + if f'/{platform}' in channel: + pos = channel.find(f'/{platform}') + channel = channel[:pos] + + if '://' not in channel: + channel = f'https://conda.anaconda.org/{channel}/{platform}' + else: + channel = f'{channel}/{platform}' + + if noarch: + channel = noarch_channel(channel, platform) + + return channel + + +def get_platform(): + """Generate a conda compatible platform-arch string + """ + system = PLATFORM.system() + machine = PLATFORM.machine() + + result = None + try: + result = '-'.join([system_map[system], machine_map[machine]]) + except KeyError: + print(f'Unknown platform/arch combination: {system}/{machine}', + file=sys.stderr) + + return result + + +def main(): + from argparse import ArgumentParser + + parser = ArgumentParser() + parser.add_argument('--benchmark', action='store_true', + help='Display total time to parse and sort channel data') + + parser.add_argument('--brute-force', action='store_true', + help='Derive timestamps from HTTP header: "last-modified"') + + parser.add_argument('--channel', '-c', default=conda_channel_pool, + action='append', dest='channels', help='Conda channel') + + parser.add_argument('--order', '-o', default='asc', help='[asc|dsc]') + + parser.add_argument('--platform', '-p', default=[get_platform()], + action='append', dest='platforms', + help=f'[{"|".join(system_map.values())}]' + f'-[{"|".join(machine_map.values())}]') + + parser.add_argument('--time-span', '-t', default='1c', + help=f'i[{"|".join([x for x in time_units.keys()])}]' + ' (120s, 12h, 1d, 2w, 3m, 4y)') + + args = parser.parse_args() + + order = False # Ascending + if args.order != 'asc': + order = True # Descending + + if args.benchmark: + timer_start = time.time() + + channels = list() + for platform in set(args.platforms): + channels.extend([convert_channel(x, platform) for x in args.channels]) + channels.extend([convert_channel(x, platform, noarch=True) + for x in args.channels]) + channels = sorted(set(channels)) + + today = datetime.now() + span_delta = today - timedelta(seconds=convert_human_timespan(args.time_span)) + packages = get_packages(channels) + timestamps = sorted(list(get_timestamps(packages, args.brute_force)), + reverse=order, key=lambda x: x['timestamp']) + + if args.benchmark: + timer_stop = time.time() + print('#benchmark: {:.02f}s'.format(timer_stop - timer_start)) + + channel_width = max([len(extract_channel_platform(x)) for x in channels]) + 1 + print('#{:<20s} {:<{channel_width}s} {:<40s}'.format( + 'date', 'channel', 'package', channel_width=channel_width)) + + try: + for info in timestamps: + name = info['name'] + ts = info['timestamp'] + chn = info['channel'] + + tstr = ts.isoformat() + if span_delta < ts: + print(f'{tstr:<20s}: {chn:<{channel_width}s}: {name:<40s}') + except IOError as e: + # Broken pipe on '|head' + # TODO: Figure out why + if e.errno == errno.EPIPE: + pass + + +if __name__ == '__main__': + main() diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..3f19155 --- /dev/null +++ b/setup.py @@ -0,0 +1,57 @@ +import os +import subprocess +import sys +from setuptools import setup, find_packages + + +PACKAGENAME='firewatch' + +if os.path.exists('relic'): + sys.path.insert(1, 'relic') + import relic.release +else: + try: + import relic.release + except ImportError: + try: + subprocess.check_call(['git', 'clone', + 'https://github.com/jhunkeler/relic.git']) + sys.path.insert(1, 'relic') + import relic.release + except subprocess.CalledProcessError as e: + print(e) + exit(1) + + +version = relic.release.get_info() +relic.release.write_template(version, PACKAGENAME) + + +setup( + name='firewatch', + version=version.pep386, + author='Joseph Hunkeler', + author_email='jhunk@stsci.edu', + description='A utility to display the timeline of a Conda channel', + license='BSD', + url='https://github.com/jhunkeler/firewatch', + classifiers=[ + 'Intended Audience :: System Administrators', + 'License :: OSI Approved :: BSD License', + 'Operating System :: OS Independent', + 'Environment :: Console', + 'Programming Language :: Python :: 3 :: Only', + 'Programming Language :: Python :: 3.6', + 'Topic :: Utilities', + 'Topic :: Software Development :: Libraries :: Python Modules', + ], + install_requires=[ + 'requests', + ], + packages=find_packages(), + entry_points={ + 'console_scripts': [ + 'firewatch=firewatch.firewatch:main' + ], + }, +) |