Add documentation

5 files changed, 122 insertions, 1 deletions

diff --git a/README b/README
new file mode 100644
index 0000000..a1ac8b3
--- /dev/null
+++ b/README
@@ -0,0 +1,24 @@
+# Extremely abridged instructions
+
+## Start htcondor
+
+### RHEL / CentOS
+Execute:
+$ service condor start
+
+### OSX
+$ /path/to/condor/sbin/condor_master
+
+
+## Generate sample data
+
+$ cd data/sample1
+$ ./mkdata.sh
+
+## Check out the generators
+
+$ cd ../../generators
+$ ./generator1.py
+$ ./generator2.py
+
+
diff --git a/generator/generator1.py b/generator/generator1.py
new file mode 100755
index 0000000..a1d61f8
--- /dev/null
+++ b/generator/generator1.py
@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+''' Will create and run MANY job files (one for each sample data file)
+'''
+import os
+from htc_utils import Job, Submit, Wait
+from glob import glob
+from pprint import pprint
+
+
+# Fill in the paths if HTCondor is not already installed globally:
+# os.environ['CONDOR_CONFIG'] = '/path/to/condor/etc/condor_config'
+# os.environ['PATH'] = ':'.join(['/path/to/condor/bin', os.environ['PATH']])
+
+
+def job_spawner(ifile):
+    print("Spawning: {}".format(ifile))
+    
+    shortname = os.path.basename(os.path.splitext(ifile)[0])
+    
+    j = Job(shortname)
+    j.logging('logs', create=True)
+    j.attr('executable', os.path.abspath('../worker/worker1.py'))
+    j.attr('arguments', '--output-dir {} {}'.format(RESULTS, ifile))
+    j.attr('queue')
+    j.commit()
+    
+    sub = Submit(j)
+    sub.execute()
+    
+    # Here's something fun --
+    # This job will block until it is finished before more data is processed
+    wait = Wait(sub)
+    
+    # If you don't want to see the output, comment the following line:
+    wait.toggle_echo()
+    wait.execute()
+    
+    
+if __name__ == '__main__':
+    RESULTS = os.path.abspath('../results/sample1')
+    DATA = [ os.path.abspath(g) for g in glob('../data/sample1/*.dat') ]
+
+    for data in DATA:
+        job_spawner(data)
\ No newline at end of file
diff --git a/generator/generator2.py b/generator/generator2.py
new file mode 100755
index 0000000..98aa142
--- /dev/null
+++ b/generator/generator2.py
@@ -0,0 +1,46 @@
+import os
+import tempfile
+from htc_utils import Job, Submit
+from glob import glob
+from pprint import pprint
+
+
+# Fill in the paths if HTCondor is not already installed globally:
+# os.environ['CONDOR_CONFIG'] = '/path/to/condor/etc/condor_config'
+# os.environ['PATH'] = ':'.join(['/path/to/condor/bin', os.environ['PATH']])
+
+def job_spawner(args):
+    ifile = tempfile.NamedTemporaryFile(mode='w+b', prefix='generator2_', dir='.', delete=True)
+    print("Spawning: {}".format(ifile.name))
+    
+    # Make sure we do not have a file extension 
+    shortname = os.path.basename(os.path.splitext(ifile.name)[0])
+    
+    # Begin job file generation
+    j = Job(shortname)
+    
+    # Enable HTCondor logging (logs stored in "./logs")
+    j.logging('logs', create=True)
+    
+    # Instruct HTCondor to run an executable with particular arguments, and queue it for
+    # each argument passed to the function
+    for arg in args:
+        j.subattr('executable', os.path.abspath('../worker/worker1.py'))
+        j.subattr('arguments', '--output-dir {} {}'.format(RESULTS, arg))
+        j.subattr('queue')
+        
+    # Save the fresh job file 
+    j.commit()
+    
+    # Submit the job file to the cluster.
+    sub = Submit(j)
+    sub.execute()
+
+
+if __name__ == '__main__':
+	# Where to save output
+    RESULTS = os.path.abspath('../results/sample1')
+    
+    # Create list of paths to our sample data (created with mkdata.sh)
+    DATA = [ os.path.abspath(g) for g in glob('../data/sample1/*.dat') ]
+    job_spawner(DATA)
diff --git a/results/.gitkeep b/results/.gitkeep
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/results/.gitkeep
diff --git a/worker/worker1.py b/worker/worker1.py
index ae2f9d8..8a31501 100755
--- a/worker/worker1.py
+++ b/worker/worker1.py
@@ -1,4 +1,11 @@
 #!/usr/bin/env python
+''' All this does is:
+    1. Open an input file
+    2. Read a list of integers (x)
+    3. Add to the integers: x += 1
+    4. Write list of integers to a new file
+'''
+
 import os
 import argparse
 
@@ -32,4 +39,4 @@ if __name__ == '__main__':
     for infile in ARGS.INFILE:
         do_work(os.path.abspath(infile))
     
-    exit(0)
\ No newline at end of file
+    exit(0)