Initial commit

1 files changed, 71 insertions, 0 deletions

diff --git a/entomb.py b/entomb.py
new file mode 100644
index 0000000..4c19b8a
--- /dev/null
+++ b/entomb.py
@@ -0,0 +1,71 @@
+#!/usr/bin/env python
+import os
+import argparse
+import requests
+import shutil
+
+
+def channel_dir(d):
+    dname = os.path.dirname(d)
+    channel = '/'.join(dname.split('/')[-2:])
+    return channel
+
+
+def download(url, destdir='.', clobber=True):
+    filename = url.split('/')[-1]
+
+    if destdir != '.':
+        os.makedirs(destdir, mode=0o775, exist_ok=True)
+
+    outfile = os.path.join(destdir, filename);
+    if not clobber and os.path.exists(outfile):
+        print("Skipping: {}".format(outfile))
+        return outfile
+
+    if not url.startswith('http'):
+        if url.startswith('file://'):
+            url = url.replace('file://', '')
+
+        print("Copying: {} -> {}".format(url, destdir))
+        shutil.copy2(url, outfile)
+        return outfile
+
+    print("Downloading: {} -> {}".format(url, destdir))
+    r = requests.get(url, stream = True)
+
+    with open(outfile,"w+b") as fp:
+        for chunk in r.iter_content(chunk_size=0xFFFF):
+            if chunk:
+                fp.write(chunk)
+    return outfile
+
+def spec_read(filename):
+    urls = []
+    with open(filename, 'r') as fp:
+        for line in fp:
+            line = line.strip()
+            if not line or line.startswith('#') or line.startswith('@'):
+                continue
+            urls.append(line)
+    return urls
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-b', '--base-dir')
+    parser.add_argument('-c', '--clobber')
+    parser.add_argument('spec_file')
+    args = parser.parse_args()
+
+    base_dir = args.base_dir
+    #spec_url = 'https://ssb.stsci.edu/releases/jwstdp/0.13.8/latest-linux'
+    spec_url = args.spec_file
+    spec_dir = os.path.join('specs', base_dir)
+
+    spec_file = download(spec_url, destdir=spec_dir)
+    urls = spec_read(spec_file)
+
+    for url in urls:
+        new_channel = os.path.join(base_dir, channel_dir(url))
+        download(url, new_channel, clobber=args.clobber);
+