CC-2784: Stopping rabbitmq-server causes pypo CPU usage to increase dramatically

-fixed

python_apps/media-monitor/airtime-media-monitor-init-d

@@ -17,13 +17,13 @@ DAEMON=/usr/lib/airtime/media-monitor/airtime-media-monitor
 PIDFILE=/var/run/airtime-media-monitor.pid
 
 start () {
-        #monit monitor airtime-media-monitor >/dev/null 2>&1
+        monit monitor airtime-media-monitor >/dev/null 2>&1
         start-stop-daemon --start --background --quiet --chuid $USERID:$GROUPID --make-pidfile --pidfile $PIDFILE --startas $DAEMON
 }
 
 stop () {
         # Send TERM after 5 seconds, wait at most 30 seconds.
-        #monit unmonitor airtime-media-monitor >/dev/null 2>&1
+        monit unmonitor airtime-media-monitor >/dev/null 2>&1
         start-stop-daemon --stop --oknodo --retry TERM/5/0/30 --quiet --pidfile $PIDFILE
         rm -f $PIDFILE
 }

python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py

@@ -2,14 +2,9 @@ import socket
 import logging
 import time
 
-
 import pyinotify
 from pyinotify import ProcessEvent
 
-# For RabbitMQ
-from kombu.connection import BrokerConnection
-from kombu.messaging import Exchange, Queue, Consumer, Producer
-
 from airtimemetadata import AirtimeMetadata
 from airtimefilemonitor.mediaconfig import AirtimeMediaConfig
 
@@ -186,8 +181,10 @@ class AirtimeProcessEvent(ProcessEvent):
             self.mmc.touch_index_file()
             
             self.file_events = []
-        #yeild to workder thread
+
+        #yield to worker thread
         time.sleep(0)
+        
         #use items() because we are going to be modifying this
         #dictionary while iterating over it.
         for k, pair in self.cookies_IN_MOVED_FROM.items():
@@ -206,7 +203,7 @@ class AirtimeProcessEvent(ProcessEvent):
                 self.handle_removed_file(False, event.pathname)
 
 
-        #check for any events recieved from Airtime.
+        #check for any events received from Airtime.
         try:
             notifier.connection.drain_events(timeout=0.1)
         #avoid logging a bunch of timeout messages.
@@ -214,4 +211,5 @@ class AirtimeProcessEvent(ProcessEvent):
             pass
         except Exception, e:
             self.logger.info("%s", e)
+            time.sleep(3)
 

python_apps/monit/airtime-monit.cfg

@@ -13,10 +13,10 @@
     with pidfile "/var/run/airtime-liquidsoap.pid"
     start program = "/etc/init.d/airtime-playout start" with timeout 10 seconds
     stop program = "/etc/init.d/airtime-playout stop"
-# check process airtime-media-monitor
+ check process airtime-media-monitor
-#    with pidfile "/var/run/airtime-media-monitor.pid"
+    with pidfile "/var/run/airtime-media-monitor.pid"
-#    start program = "/etc/init.d/airtime-media-monitor start" with timeout 10 seconds
+    start program = "/etc/init.d/airtime-media-monitor start" with timeout 10 seconds
-#    stop program = "/etc/init.d/airtime-media-monitor stop"
+    stop program = "/etc/init.d/airtime-media-monitor stop"
  check process airtime-show-recorder
     with pidfile "/var/run/airtime-show-recorder.pid"
     start program = "/etc/init.d/airtime-show-recorder start" with timeout 10 seconds

python_apps/pypo/pypofetch.py

@@ -9,6 +9,7 @@ import string
 import json
 import telnetlib
 import math
+import socket
 from threading import Thread
 from subprocess import Popen, PIPE
 from datetime import datetime
@@ -351,12 +352,7 @@ class PypoFetch(Thread):
                     logger.error(e)
 
 
-    """
+    def main(self):
-    Main loop of the thread:
-    Wait for schedule updates from RabbitMQ, but in case there arent any,
-    poll the server to get the upcoming schedule.
-    """
-    def run(self):
         logger = logging.getLogger('fetch')
 
         while not self.init_rabbit_mq():
@@ -384,12 +380,31 @@ class PypoFetch(Thread):
                 # Hooray for globals!
                 schedule_data = SCHEDULE_PUSH_MSG
                 status = 1
-            except:    
+            except socket.timeout, se:
                 # We didnt get a message for a while, so poll the server
                 # to get an updated schedule. 
                 status, schedule_data = self.api_client.get_schedule()
+            except Exception, e:
+                """
+                This Generic exception is thrown whenever the RabbitMQ
+                Service is stopped. In this case let's check every few
+                seconds to see if it has come back up
+                """
+                logger.info("Unknown exception")
+                return
+
+            #return based on the exception
             
             if status == 1:
                 self.process_schedule(schedule_data, "scheduler", False)                
-            loops += 1
+            loops += 1        
+
+    """
+    Main loop of the thread:
+    Wait for schedule updates from RabbitMQ, but in case there arent any,
+    poll the server to get the upcoming schedule.
+    """
+    def run(self):
+        while True:
+            self.main()
             

python_apps/show-recorder/recorder.py

@@ -7,6 +7,7 @@ import datetime
 import os
 import sys
 import shutil
+import socket
 
 from configobj import ConfigObj
 
@@ -289,10 +290,12 @@ class CommandListener(Thread):
             try:
                 # block until 5 seconds before the next show start
                 self.connection.drain_events(timeout=self.time_till_next_show)
-            except Exception, e:
+            except socket.timeout, s:
-                self.logger.info(e)
+                self.logger.info(s)
                 # start recording
                 self.start_record()
+            except Exception, e:
+                time.sleep(3)
 
             loops += 1