blob: a1d61f897783d4a94ac1ec059a5a6a1c018890f4 (
plain) (
blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
|
#!/usr/bin/env python
''' Will create and run MANY job files (one for each sample data file)
'''
import os
from htc_utils import Job, Submit, Wait
from glob import glob
from pprint import pprint
# Fill in the paths if HTCondor is not already installed globally:
# os.environ['CONDOR_CONFIG'] = '/path/to/condor/etc/condor_config'
# os.environ['PATH'] = ':'.join(['/path/to/condor/bin', os.environ['PATH']])
def job_spawner(ifile):
print("Spawning: {}".format(ifile))
shortname = os.path.basename(os.path.splitext(ifile)[0])
j = Job(shortname)
j.logging('logs', create=True)
j.attr('executable', os.path.abspath('../worker/worker1.py'))
j.attr('arguments', '--output-dir {} {}'.format(RESULTS, ifile))
j.attr('queue')
j.commit()
sub = Submit(j)
sub.execute()
# Here's something fun --
# This job will block until it is finished before more data is processed
wait = Wait(sub)
# If you don't want to see the output, comment the following line:
wait.toggle_echo()
wait.execute()
if __name__ == '__main__':
RESULTS = os.path.abspath('../results/sample1')
DATA = [ os.path.abspath(g) for g in glob('../data/sample1/*.dat') ]
for data in DATA:
job_spawner(data)
|
