Add delay when respawning dead minions

pikatasks/worker.py

@@ -11,6 +11,7 @@ from . import django_compat
 
 
 IPC_PERIOD = timedelta(seconds=0.2)  # how often processes check their signals and other processes
+RESPAWN_DEAD_MINIONS_DELAY = timedelta(seconds=10)  # avoid converting CPU time directly into gigabytes of logs when minions die on start (network failures etc...)
 
 
 class SignalStopHandler:
@@ -54,6 +55,7 @@ def start(tasks="all", number_of_processes=None, worker_stop_handler=None):
     """
 
     logger = logging.getLogger("pikatasks.worker.master")
+    number_of_processes = number_of_processes or settings.WORKER_TASK_PROCESSES
     processes = list()
 
     def remove_ended_processes():
@@ -136,10 +138,13 @@ def start(tasks="all", number_of_processes=None, worker_stop_handler=None):
         raise ValueError("Empty task list.")
     if not worker_stop_handler:
         worker_stop_handler = SignalStopHandler(logger=logger, this_process_name="master")
+    minions_respawned_at = datetime.utcnow() - RESPAWN_DEAD_MINIONS_DELAY * 2
     while not worker_stop_handler.stop_is_requested:
         remove_ended_processes()
-        while len(processes) < (number_of_processes or settings.WORKER_TASK_PROCESSES):
+        if datetime.utcnow() > minions_respawned_at + RESPAWN_DEAD_MINIONS_DELAY:
+            while len(processes) < number_of_processes:
                 processes.append(create_minion(tasks))
+                minions_respawned_at = datetime.utcnow()
         time.sleep(IPC_PERIOD.total_seconds())
     # stopping
     logger.info("Stopping minions...")