''' Stuff related to the tree structure of the command set ''' import fnmatch #import exyapps.runtime ##### ##### class command_tree(object): # a dict that maps currently known node names to node objects node_index = None # This is a list if (before, after) showing names of node orders; # the individual nodes do not store this information while the # parsing is happening. node_order = None # call init() once before parsing, then call parse for each file, then call finish() def __init__( self ) : self.node_index = { } self.node_order = [ ] def finish( self ) : # This links up the nodes according to the (before, after) information # in node_order. before and after are fully qualified node names. # We do this at the end so that # - we can have forward references # - we can use wild cards in our AFTER clauses # before is the predecessor, after comes AFTER the predecessor for before, after, required, pos in self.node_order : if ( '*' in before ) or ( '?' in before ) or ( '[' in before ): host, table, cmd = crack_name(before) for x in self.node_index : hl, tl, cl = crack_name(x) if ( fnmatch.fnmatchcase(hl,host ) and fnmatch.fnmatchcase(tl,table) and fnmatch.fnmatchcase(cl,cmd ) ) : # yes, the wild card matches this one; connect them # (but don't let a node come before itself because the wild card is too loose) if x != after : self.connect(x, after, required, pos) else : self.connect(before, after, required, pos) # Work out the depths of each node. # Since we are walking the graph, we also detect dependency loops here. compute_depths( self.node_index ) # make the actual connection between nodes def connect( self, before, after, required, line ) : if not before in self.node_index : if required : print "error: %s happens after non-existant %s - line %s"%(before,after,line) return if not after in self.node_index : print "error: after node %s does not exist %s"%(after,line) return # tell the after node that the other one comes first self.node_index[after].predecessors.append(self.node_index[before]) # tell the after node about the before node and note that # the before node is not done yet. self.node_index[after].released[before] = False # create a set of nodes for a particular command - called from within the parser def add_command_list( self, table, hostlist, command_list ) : for host in hostlist : this_table = '%s:%s' % ( host, table ) for command, script, script_type, after_clause, pos in command_list : # this happens once for each CMD clause # command is the name of this command # script is the script to run # after is a list of AFTER clauses # pos is where in this file this command was defined command = normalize_name( host, table, command ) if command in self.node_index : # bug: should be error print "# warning: %s already used on line %s"%(command,self.node_index[command].input_line) # create the node self.node_index[command]=node(command, script, script_type, nice_pos( current_file_name, pos) ) for before_name, required, pos in after_clause : # this happens once for each AFTER clause # before is the name of a predecessor that this one comes after # required is a boolean, whether the predecessor must exist # pos is where in the file the AFTER clause begins before_name = normalize_name( host, table, before_name ) # list of ( before, after, required, pos ) self.node_order.append( (before_name, command, required, pos) ) ##### # crack open host:table/cmd def crack_name(name) : if ':' in name : t = name.split(':') host = t[0] name = t[1] else : host = '*' if '/' in name : t = name.split('/') table = t[0] cmd = t[1] else : table = '*' cmd = name return (host, table, cmd) ##### def join_name( host, table, cmd ) : return '%s:%s/%s'%(host,table,cmd) ##### def normalize_name( host, table, name ) : if not '/' in name : name = table + '/' + name if name.startswith('.:') : name = name[2:] if not ':' in name : name = host + ':' + name # print "## return %s"%name return name ##### _wildcard_cache = ( None, None ) def wildcard_name( wild, name ) : global _wildcard_cache if wild != _wildcard_cache : host, table, cmd = crack_name(wild) _wildcard_cache = ( name, ( host, table, cmd ) ) host, table, cmd = _wildcard_cache[1] hl, tl, cl = crack_name(name) return ( fnmatch.fnmatchcase(hl,host ) and fnmatch.fnmatchcase(tl,table) and fnmatch.fnmatchcase(cl,cmd ) ) ##### ##### # a node object for each command instance that will be run. The name is # "host:table/command". The shell command to execute on the target host # is "script". # # predecessors[] is a list of everything that this node must definitely come after. # # released is a dict indexed by each of the "before" nodes; the value is # true if the before node is finished running. # class node(object) : def __init__(self, name, script, script_type, input_line) : # the fully qualified name of the node self.name = name # the command script that this node runs self.script = script self.script_type = script_type # what line of the input file specified this node; this is # a string of the form "foo.bar 123" self.input_line = input_line # crack open host:table/cmd self.host, self.table, self.cmd = crack_name(name) # this command runs after every node in this list self.predecessors = [ ] # this is a dict of everything that comes before us # The key is the name of the other node. # The value is true/false for whether the other node # is finished running. self.released = { } # These flags are 1 or 0 so we can sum() them self.finished = 0 self.running = 0 ##### # debug - make a string representation of all the nodes def show_nodes( node_index ) : import cStringIO as StringIO s = StringIO.StringIO() for x in sorted( [ x for x in node_index ] ) : x = node_index[x] # show the node name and the command it runs s.write( "NODE %s %s %s\n"%(x.name, x.script, x.input_line) ) # show each node that this one comes After, and show the flag # whether the previous node has run yet. for y in x.predecessors : s.write( " AFTER %s\n"%y.name ) return s.getvalue() ##### def nice_pos( filename, yapps_pos ) : # filename is the file name to use # yapps_pos is a tuple of (file, line, col) except that file # is a file-like object with no traceability to an actual file1 return "%s:%s col %s"%(filename, yapps_pos[1], yapps_pos[2]) ##### ##### # # calculating depths of a node tree def c_d_fn(x,depth) : if x.in_recursion : print "error: loop detected at",x.name return # if it is already deeper than where we are now, we can (must) # prune the tree walk here. if x.depth >= depth : return if depth > 100 : # bug: proxy for somebody wrote a loop print "error: depth > 100, node = ",x.name return x.in_recursion = 1 # assign the depth x.depth = depth # make all the successors one deeper depth = depth + 1 for y in x.successors : c_d_fn(y,depth) x.in_recursion = 0 def compute_depths(nodes) : # init everything for x in nodes : x = nodes[x] x.depth = 0 x.successors = [ ] x.in_recursion = 0 # walk the nodes in an arbitrary order; make a list of successors for x in nodes : x = nodes[x] for y in x.predecessors : if not x in y.successors : y.successors.append(x) # recursively walk down the tree assigning depth values, starting # with depth=1 for the highest level for x in nodes : c_d_fn(nodes[x],1) # for x in nodes : x = nodes[x] del x.in_recursion ##### import specfile current_file_name = None def read_file_list( file_list ) : global current_file_name di = command_tree( ) for x in file_list : current_file_name = x sc = specfile.specfileScanner( open(x,'r').read() ) p = specfile.specfile( scanner=sc, data=di ) result = specfile.wrap_error_reporter( p, 'start' ) di.finish() return di if __name__=='__main__': import sys n = read_file_list( sys.argv[1:] ) print show_nodes(n.node_index)