checkpoint

git-svn-id: https://svn.stsci.edu/svn/ssb/etal/steuermann/trunk@421 d34015c8-bcbb-4646-8ac8-8ba5febf221d

9 files changed, 116 insertions, 80 deletions

diff --git a/README b/README
index 8b652b7..b0cd1d7 100644
--- a/README
+++ b/README
@@ -7,3 +7,7 @@ make
 
 python setup.py install
 
+smc test.sm
+?
+
+
diff --git a/daily.sr b/daily.sm
index eb0c691..eb0c691 100644
--- a/daily.sr
+++ b/daily.sm
diff --git a/steuermann/hosts.ini b/steuermann/hosts.ini
index c64e2aa..d51e9df 100644
--- a/steuermann/hosts.ini
+++ b/steuermann/hosts.ini
@@ -12,54 +12,67 @@
 ;	%(cmd)s     -
 ;	%(foo)s	    - if you include a foo= line
 ;
+
+; definitions common to various operating system environments
+
+[all]
+local=[ 'sh', '-c', '%(script)s' ]
+
 [linux:csh]
 hostname=no_such_machine
 run=[ 'ssh', '-q', '%(hostname)s', 'source .steuermann.%(hostname)s; cd %(workdir)s; hostname; %(script)s ' ]
-workdir=/tmp
+like=all
+
+[windows:cmd]
+hostname=no_such_machine
+run=[ 'python', '-m', 'steuermann.windows_comm', '%(hostname)s', '%(script)s' ]
+like=all
+
 
+; machines defined named after each OS
 
 [rhe4-32]
-hostname=herbert
-like=linux:csh
+like=herbert
 
 [rhe4-64]
-hostname=thor
-like=linux:csh
+like=thor
 
 [rhe5-64]
-hostname=arzach
-like=linux:csh
+like=arzach
 
 [leopard]
-hostname=bond
-like=linux:csh
+like=bond
 
 [snow-leopard]
-hostname=cadeau
-like=linux:csh
+like=cadeau
 
 
-[arzach]
-hostname=arzach
+; actual machines
+
+[herbert]
+hostname=herbert
 like=linux:csh
+workdir=/herbert/data1/sienkiew/steuermann
 
 [thor]
 hostname=thor
 like=linux:csh
+workdir=/thor/data2/sienkiew/steuermann
 
+[arzach]
+hostname=arzach
+like=linux:csh
+workdir=/arzach/data1/sienkiew/steuermann
 
-[host_a]
-run=[ 'sleep', '3' ]
-
-[host_b]
-run=[ 'sleep', '3' ]
-
-[host_c]
-run=[ 'sleep', '3' ]
-
-[host_d]
-run=[ 'sleep', '3' ]
+[bond]
+hostname=bond
+like=linux:csh
+workdir=/Users/sienkiew/work/steuermann
 
+[cadeau]
+hostname=cadeau
+like=linux:csh
+workdir=/Users/sienkiew/work/steuermann
 
 ; There is a section [ALL] that is used with every machine name
 [ALL]
diff --git a/steuermann/nodes.py b/steuermann/nodes.py
index dfdb6d7..0173793 100644
--- a/steuermann/nodes.py
+++ b/steuermann/nodes.py
@@ -81,7 +81,7 @@ class command_tree(object):
     def add_command_list( self, table, hostlist, command_list ) :
         for host in hostlist :
             this_table = '%s:%s' % ( host, table )
-            for command, script, after_clause, pos in command_list :
+            for command, script, script_type, after_clause, pos in command_list :
                 # this happens once for each CMD clause
                 # command is the name of this command
                 # script is the script to run
@@ -95,7 +95,7 @@ class command_tree(object):
                     print "# warning: %s already used on line %s"%(command,self.node_index[command].input_line)
 
                 # create the node
-                self.node_index[command]=node(command, script, nice_pos( current_file_name, pos)  )
+                self.node_index[command]=node(command, script, script_type, nice_pos( current_file_name, pos)  )
 
                 for before_name, required, pos in after_clause :
                     # this happens once for each AFTER clause
@@ -178,12 +178,13 @@ def wildcard_name( wild, name ) :
 # true if the before node is finished running.
 #
 class node(object) :
-    def __init__(self, name, script, input_line) :
+    def __init__(self, name, script, script_type, input_line) :
         # the fully qualified name of the node
         self.name = name
 
         # the command script that this node runs
         self.script = script
+        self.script_type = script_type
 
         # what line of the input file specified this node; this is
         # a string of the form "foo.bar 123"
diff --git a/steuermann/report.py b/steuermann/report.py
index e1d2971..ca8d064 100644
--- a/steuermann/report.py
+++ b/steuermann/report.py
@@ -91,7 +91,8 @@ def get_table( db, run_name, tablename, info_callback, showdepth=0 ) :
         if cmd != prev_cmd :
             row = row + 1
             t.set_value(row, 0, cmd)
-            t.set_value(row, 'depth', depth)
+            if showdepth :
+                t.set_value(row, 'depth', depth)
             prev_cmd = cmd
             
         info = info_callback( db, run_name, tablename, host, cmd )
diff --git a/steuermann/run.py b/steuermann/run.py
index f3e86b3..1379120 100644
--- a/steuermann/run.py
+++ b/steuermann/run.py
@@ -23,24 +23,26 @@ class struct :
 class runner(object): 
 
     # dict of all current running processes, indexed by node name
-    all_procs = { }
+    all_procs = None
 
     # index of nodes
-    node_index = { }
-
-    # info about hosts we can run on
-    host_info = { }
+    node_index = None
 
     # dir where we write our logs
     logdir = ''
 
+    # 
+    host_info_cache = None
+
     #####
     #
 
     def __init__( self, nodes, logdir ) :
+        self.all_procs = { }
         self.node_index = nodes
         self.load_host_info()
         self.logdir = logdir
+        self.host_info_cache = { }
 
     #####
     # start a process
@@ -53,16 +55,16 @@ class runner(object):
             print "....%s:%s/%s\n"%(node.host, node.table, node.cmd)
 
         try :
-            args = self.host_info[node.host]
+            args = self.get_host_info(node.host)
         except :
             print "ERROR: do not know how to run on %s"%node.host
             raise
 
-        run = args['run']
 
         args = args.copy()
         args.update( 
             script=node.script,
+            script_type=node.script_type,
             host=node.host,
             table=node.table,
             cmd=node.cmd
@@ -73,7 +75,21 @@ class runner(object):
             for x in sorted([x for x in args]) :
                 print '%s=%s'%(x,args[x])
 
-        run = [ x % args for x in run ]
+        args['script'] = args['script'] % args
+
+        if args['script_type'] == 'r' :
+            run = args['run']
+        elif  args['script_type'] == 'l' :
+            run = args['local']
+        else :
+            raise Exception()
+
+        t = [ ]
+        for x in run :
+            # bug: what to do in case of keyerror
+            t.append( x % args )
+
+        run = t
 
         if debug :
             print "RUN",run
@@ -169,53 +185,37 @@ class runner(object):
     #####
 
 
-    def _host_get_names( self, cfg, section, d ) :
+    def _host_get_names( self, cfg, section ) :
+        d = { }
         # pick all the variables out of this section
         for name, value in cfg.items(section) :
-            if name == 'run' :
-                # run is a list
+            if value.startswith('[') :
+                # it is a list
                 d[name] = eval(value)
             else :
                 # everything else is plain text
                 d[name] = value
+        return d
 
     def load_host_info( self, filename=None ) : 
-        self.host_info = { }
 
         # read the config file
         if filename is None :
             filename = os.path.dirname(__file__) + '/hosts.ini'
-        cfg = ConfigParser.RawConfigParser()
-        cfg.read(filename)
-
-	    # this dict holds the set of values that are defined as
-	    # applying to all hosts
-        all_dict = { }
-        self._host_get_names(cfg, 'ALL', all_dict)
+        self.cfg = ConfigParser.RawConfigParser()
+        self.cfg.read(filename)
 
-        # for all the sections (except ALL) get the names from that section
-        for x in cfg.sections() :
-            if x == 'ALL' :
-                continue
+    def get_host_info(self, host) :
+        if not host in self.host_info_cache :
 
-            # start with a dict that contains what is in ALL
-            d = all_dict.copy()
-
-            # get what there is to know about host x
-            self._host_get_names(cfg, x, d)
-
-            # if it is like some other host, start over using ALL, then
-            # the LIKE host, then our own information
+            d = self._host_get_names(self.cfg, host)
             if 'like' in d :
-                like = d['like']
-                d = all_dict.copy()
-                self._host_get_names(cfg, like, d)
-                self._host_get_names(cfg, x, d)
+                d1 = self.get_host_info(d['like'])
                 del d['like']
+                d1.update(d)
+                self.host_info_cache[host] = d1
+            else :
+                self.host_info_cache[host] = d
 
-            print x,d
-            self.host_info[x] = d
-
-        del cfg
-
+        return self.host_info_cache[host]
     #####
diff --git a/steuermann/run_all.py b/steuermann/run_all.py
index d94cafa..38f8241 100644
--- a/steuermann/run_all.py
+++ b/steuermann/run_all.py
@@ -37,15 +37,18 @@ def main() :
 
     #
 
-    di_nodes = nodes.read_file_list( sys.argv[2:] )
+    all = sys.argv[1] == '-a' 
+    if all :
+        di_nodes = nodes.read_file_list( sys.argv[2:] )
+    else :
+        di_nodes = nodes.read_file_list( sys.argv[1:] )
 
     xnodes = di_nodes.node_index
     run_name = str(datetime.datetime.now()).replace(' ','_')
     db = steuermann.config.open_db()
     register_database(db, run_name, xnodes)
 
-    n = sys.argv[1]
-    if n == '-a' :
+    if all :
         run_all(xnodes, run_name, db)
     else :
         run_interactive( xnodes, run_name, db )
@@ -55,6 +58,8 @@ def main() :
 def do_flag( xnodes, name, recursive, fn, verbose ) :
     if verbose :
         verbose = verbose + 1
+    if not (':' in name ) and not ('/' in name) :
+        name = '*:*/'+name
     if not ':' in name :
         name = '*:' + name
     if ( '*' in name )  or ( '?' in name ) or ( '[' in name ) :
@@ -180,7 +185,6 @@ def run_interactive( xnodes, run_name, db) :
                 if keypress() :
                     print "wait interrupted (processes continue)"
                     break
-            print "wait finished"
 
         if keep_running :
             print "run step"
@@ -279,7 +283,6 @@ def run_step( runner, xnodes, run_name, db ) :
         no_sleep = 0
 
         # Loop, polling for work to do, or for finishing processes
-        print "loop"
         for x_name in xnodes :
             x=xnodes[x_name]
 
@@ -317,7 +320,7 @@ def run_step( runner, xnodes, run_name, db ) :
             # of predecessors, we can run this one
             if released == len(x.released) :
                 host, table, cmd = nodes.crack_name(x_name)
-                print "RUN", x_name
+                # print "RUN NODE", x_name
 
                 db.execute("UPDATE status SET start_time = ?, status = 'S' WHERE ( run = ? AND host = ? AND tablename = ? AND cmd = ? )",
                     ( str(datetime.datetime.now()), run_name, host, table, cmd ) )
@@ -330,7 +333,7 @@ def run_step( runner, xnodes, run_name, db ) :
             if not who_exited :
                 break
 
-            print "SOMETHING EXITED",who_exited
+            # print "SOMETHING EXITED",who_exited
             # yes, something exited - no sleep, and keep running
             no_sleep = 1
             keep_running = 1
@@ -344,7 +347,7 @@ def run_step( runner, xnodes, run_name, db ) :
                     ( str(datetime.datetime.now()), who_exited[1], run_name, x_host, x_table, x_cmd ) )
             db.commit()
 
-        runner.display_procs()
+        # runner.display_procs()
 
         return ( keep_running, no_sleep )
 
@@ -365,7 +368,7 @@ def keypress() :
 
 #####
 
-def info_callback_want( tablename, cmd, host, status ) :
+def info_callback_want( db, run, tablename, host, cmd ) :
     n = xnodes['%s:%s/%s'%(host,tablename,cmd)]
     s = ''
     if n.skip :
@@ -376,7 +379,7 @@ def info_callback_want( tablename, cmd, host, status ) :
         s = '-'
     return s
 
-def info_callback_depth( tablename, cmd, host, status ) :
+def info_callback_depth( db, run, tablename, host, cmd ) :
     n = xnodes['%s:%s/%s'%(host,tablename,cmd)]
     return n.depth
 
diff --git a/steuermann/specfile.exy b/steuermann/specfile.exy
index 8dd42a3..5dc715f 100644
--- a/steuermann/specfile.exy
+++ b/steuermann/specfile.exy
@@ -20,6 +20,7 @@ parser specfile:
     token tablename: "[a-zA-Z0-9_.-]+"
     token wildname: "[*?a-zA-Z0-9_.-]+"
     token RUN: "RUN"
+    token LOCAL: "LOCAL"
     token string: '"[^"]*"'
     token SLASH: "/"
     token COLON: ":"
@@ -49,9 +50,9 @@ parser specfile:
     rule command: 
 	# a single command, including any number of AFTER clauses
         CMD  {{ cmd_pos = self._scanner.get_pos() }} cmdname  {{ cmd_name=cmdname; script=cmdname; x_after_clause = [ ] }} 
-            [ RUN string {{ script = string[1:-1] }} ]
+            [ RUN string {{ script = string[1:-1]; script_type='r' }} | LOCAL string {{ script = string[1:-1]; script_type='l' }} ]
             ( {{ after_pos = self._scanner.get_pos() }} AFTER optword after_spec {{ x_after_clause.append( (after_spec,optword, after_pos) ) }}  )*
-            {{ return ( cmd_name, script, x_after_clause, cmd_pos ) }}  
+            {{ return ( cmd_name, script, script_type, x_after_clause, cmd_pos ) }}  
 
     # in the AFTER clause, you can say OPT to mean the node is optional (not an error if it does not exist)
     rule optword:
diff --git a/tests/hosts.py b/tests/hosts.py
new file mode 100644
index 0000000..5fbfa25
--- /dev/null
+++ b/tests/hosts.py
@@ -0,0 +1,13 @@
+import steuermann.run as run
+
+r = run.runner(0,0)
+
+def test_1() :
+    print r.cfg.sections()
+
+def test_2() :
+    print r.get_host_info('rhe4-32')
+
+
+test_1()
+test_2()