Merge branch 'cc-5709-airtime-analyzer' into saas

python_apps/airtime_analyzer/airtime_analyzer/message_listener.py

@@ -120,7 +120,12 @@ class MessageListener:
 
     def disconnect_from_messaging_server(self):
         '''Stop consuming RabbitMQ messages and disconnect'''
+        # If you try to close a connection that's already closed, you're going to have a bad time.
+        # We're breaking EAFP because this can be called multiple times depending on exception
+        # handling flow here.
+        if not self._channel.is_closed and not self._channel.is_closing:
             self._channel.stop_consuming()
+        if not self._connection.is_closed and not self._connection.is_closing:
             self._connection.close()
    
     def graceful_shutdown(self, signum, frame):

python_apps/airtime_analyzer/airtime_analyzer/status_reporter.py

@@ -57,7 +57,8 @@ def process_http_requests(ipc_queue, http_retry_queue_path):
         logging.error("Failed to unpickle %s. Continuing..." % http_retry_queue_path)
         pass
 
-    
+    while True:
+        try:
             while not shutdown:
                 try:
                     request = ipc_queue.get(block=True, timeout=5)
@@ -84,6 +85,14 @@ def process_http_requests(ipc_queue, http_retry_queue_path):
             # while the web server is down or unreachable.
             with open(http_retry_queue_path, 'wb') as pickle_file:
                 pickle.dump(retry_queue, pickle_file)
+        except Exception as e: # Terrible top-level exception handler to prevent the thread from dying, just in case.
+            if shutdown:
+                return
+            logging.exception("Unhandled exception in StatusReporter")
+            logging.exception(e)
+            logging.info("Restarting StatusReporter thread")
+            time.sleep(2) # Throttle it
+
 
 def send_http_request(picklable_request, retry_queue):
     if not isinstance(picklable_request, PicklableHttpRequest):