Initial commit

4 files changed, 74 insertions, 0 deletions

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..a9af213
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,2 @@
+.project
+.pydevproject
diff --git a/data/sample1/mkdata.sh b/data/sample1/mkdata.sh
new file mode 100644
index 0000000..e8ac434
--- /dev/null
+++ b/data/sample1/mkdata.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+for i in `seq 0 100`
+do
+	export filename="d${i}.dat"
+	for o in `seq 0 1000`
+	do
+		echo $o >> $filename
+	done
+done
+
diff --git a/generator/generator.py b/generator/generator.py
new file mode 100644
index 0000000..4088be6
--- /dev/null
+++ b/generator/generator.py
@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+import os
+from htc_utils import Job, Submit
+from multiprocessing import Pool
+from glob import glob
+from pprint import pprint
+
+os.environ['PATH'] = '/Users/jhunk/Downloads/condor-8.2.6-x86_64_MacOSX7-stripped/bin:/Users/jhunk/Downloads/condor-8.2.6-x86_64_MacOSX7-stripped/sbin:/usr/local/lib/wxPython/bin:/Users/jhunk/.virtualenvs/python27/bin:/opt/local/libexec/gnubin:/opt/local/bin:/opt/local/sbin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/opt/X11/bin:/usr/texbin'
+
+
+def job_spawner(ifile):
+    print "Spawning: {}".format(ifile)
+    j = Job(os.path.basename(os.path.splitext(ifile)[0]))
+    j.attr('executable', os.path.abspath('../worker/worker1.py'))
+    j.attr('arguments', '--output-dir {} {}'.format(RESULTS, ifile))
+    j.commit()
+    
+    
+if __name__ == '__main__':
+    RESULTS = os.path.abspath('../results/sample1')
+    DATA = [ os.path.abspath(g) for g in glob('../data/sample1/*.dat') ]
+    
+    
+    
+    p = Pool(4)
+    print(p.map(job_spawner, DATA))
\ No newline at end of file
diff --git a/worker/worker1.py b/worker/worker1.py
new file mode 100644
index 0000000..5c474b2
--- /dev/null
+++ b/worker/worker1.py
@@ -0,0 +1,36 @@
+#!/usr/bin/env python
+import os
+import argparse
+
+PARSER = argparse.ArgumentParser()
+PARSER.add_argument('--output-dir', '-o', default=os.path.abspath(os.curdir))
+PARSER.add_argument('INFILE', action='store', nargs='*', help='Input file')
+ARGS = PARSER.parse_args()
+
+def do_work(ifile):
+    ofile = os.path.join(ARGS.output_dir, os.path.basename(ifile))
+    in_data = []
+    out_data = []
+    
+    print('Loading {} ({} bytes)'.format(ifile, os.path.getsize(ifile)))
+    
+    with open(ifile) as fp:
+        for line in fp:
+            line = line.rstrip()
+            in_data.append(line)
+    
+    with open(ofile) as fp:
+        for value in in_data:
+            fp.writeline(value+1)
+            
+    
+
+if __name__ == '__main__':
+        
+    for infile in ARGS.INFILE:
+        do_work(os.path.abspath(infile))
+    else:
+        print("No input file(s) received!")
+        exit(1)
+    
+    exit(0)
\ No newline at end of file