From ed494ac58725f3ac6163882389efd14e5261acb6 Mon Sep 17 00:00:00 2001 From: Albert Santoni Date: Mon, 5 May 2014 18:25:47 -0400 Subject: [PATCH] Some defensive code against multiprocess related deadlocks * Reinitialize logging in child processes so we don't inherit locks. Might be causing a deadlock we're seeing on Pro right now. --- .../airtime_analyzer/analyzer_pipeline.py | 12 +++++++++++- .../airtime_analyzer/message_listener.py | 4 ++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/python_apps/airtime_analyzer/airtime_analyzer/analyzer_pipeline.py b/python_apps/airtime_analyzer/airtime_analyzer/analyzer_pipeline.py index 3a1465b7b..4f20f9c18 100644 --- a/python_apps/airtime_analyzer/airtime_analyzer/analyzer_pipeline.py +++ b/python_apps/airtime_analyzer/airtime_analyzer/analyzer_pipeline.py @@ -30,6 +30,10 @@ class AnalyzerPipeline: temporary randomly generated name, which is why we want to know what the original name was. """ + # Might be super critical to initialize a separate log file here so that we + # don't inherit logging/locks from the parent process. Supposedly + # this can lead to Bad Things (deadlocks): http://bugs.python.org/issue6721 + AnalyzerPipeline.setup_logging() try: if not isinstance(queue, multiprocessing.queues.Queue): raise TypeError("queue must be a multiprocessing.Queue()") @@ -59,4 +63,10 @@ class AnalyzerPipeline: logging.exception(e) raise e - + @staticmethod + def setup_logging(): + _LOG_PATH = "/var/log/airtime/airtime_analyzer_pipeline.log" + FORMAT = "%(asctime)s [%(module)s] [%(levelname)-5.5s] %(message)s" + logging.basicConfig(filename=_LOG_PATH,level=logging.DEBUG, format=FORMAT) + #rootLogger = logging.getLogger() + #rootLogger.setFormatter(logFormatter) diff --git a/python_apps/airtime_analyzer/airtime_analyzer/message_listener.py b/python_apps/airtime_analyzer/airtime_analyzer/message_listener.py index 9939f11bc..97613e81f 100644 --- a/python_apps/airtime_analyzer/airtime_analyzer/message_listener.py +++ b/python_apps/airtime_analyzer/airtime_analyzer/message_listener.py @@ -212,6 +212,10 @@ class MessageListener: logging.info(results) else: raise Exception("Analyzer process terminated unexpectedly.") + + # Ensure our queue doesn't fill up and block due to unexpected behaviour. Defensive code. + while not q.empty(): + q.get() return results