Initial "kill all tasks" work.HEADmaster

1 files changed, 9 insertions, 1 deletions

diff --git a/steuermann/run_all.py b/steuermann/run_all.py
index 2be9214..03385bf 100644
--- a/steuermann/run_all.py
+++ b/steuermann/run_all.py
@@ -462,6 +462,7 @@ def run_all(xnodes, run_name, hosts_ini, db) :
 
     while 1 :
         ( runner, keep_running, no_sleep ) = run_step( runner, xnodes, run_name, db )
+
         if not keep_running :
             break
         if not no_sleep :
@@ -596,8 +597,14 @@ def run_step( runner, xnodes, run_name, db ) :
             if not who_exited:
                 break
 
+            if who_exited[1] != 0:
+                print('TASK FAILURE: {0} (exit: {1})'.format(who_exited[0], who_exited[1]))
+                for task, knodes in runner.node_index.items():
+                    print('TASK SKIPPED: {0} (on behalf of {1})'.format(task, who_exited[0]))
+                    knodes.skip = 1
+
             # something exited; no sleep, keep running
-            print("SOMETHING EXITED", who_exited)
+            print("TASK EXITED: {0}".format(who_exited[0]))
             no_sleep = 1
             keep_running = 1
 
@@ -648,6 +655,7 @@ def run_step( runner, xnodes, run_name, db ) :
                     ( str(datetime.datetime.now()), who_exited[1], logs_exist, run_name, x_host, x_table, x_cmd ) )
             db.commit()
 
+
         return ( runner, keep_running, no_sleep )
 
 #####