CC-2977: Never delete files from the database

- using mtab(previous and current) files to figure out directories to be
added or removed
- adding index to cc_files
- see the tickets and media monitor design page for more info

python_apps/api_clients/api_client.py

@@ -592,15 +592,18 @@ class AirTimeApiClient(ApiClientInterface):
     """
     This function updates status of mounted file system information on airtime
     """
-    def update_file_system_mount(self, mount_list):
+    def update_file_system_mount(self, added_dir, removed_dir):
         logger = logging.getLogger()
         try:
             url = "http://%s:%s/%s/%s" % (self.config["base_url"], str(self.config["base_port"]), self.config["api_base"], self.config["update_fs_mount"])
             
             url = url.replace("%%api_key%%", self.config["api_key"])
 
-            data_string = string.join(mount_list, ',')
-            map = [("mount_list", data_string)]
+            added_data_string = string.join(added_dir, ',')
+            removed_data_string = string.join(removed_dir, ',')
+            
+            map = [("added_dir", added_data_string),("removed_dir",removed_data_string)]
+            
             data = urllib.urlencode(map)
             
             req = urllib2.Request(url, data)
@@ -614,7 +617,7 @@ class AirTimeApiClient(ApiClientInterface):
     
     """
         When watched dir is missing(unplugged or something) on boot up, this function will get called
-        and will call approperiate function on Airtime.
+        and will call appropriate function on Airtime.
     """
     def handle_watched_dir_missing(self, dir):
         logger = logging.getLogger()

python_apps/media-monitor/airtimefilemonitor/airtimemediamonitorbootstrap.py

@@ -1,6 +1,7 @@
 import os
 import time
 import pyinotify
+import shutil
 
 from subprocess import Popen, PIPE
 from api_clients import api_client
@@ -22,8 +23,13 @@ class AirtimeMediaMonitorBootstrap():
         self.wm = wm
         # add /etc on watch list so we can detect mount
         self.mount_file = "/etc"
+        self.curr_mtab_file = "/var/tmp/airtime/media-monitor/currMtab"
         self.logger.info("Adding %s on watch list...", self.mount_file)
         self.wm.add_watch(self.mount_file, pyinotify.ALL_EVENTS, rec=False, auto_add=False)
+        
+        # create currMtab file if it's the first time
+        if not os.path.exists(self.curr_mtab_file):
+            shutil.copy('/etc/mtab', self.curr_mtab_file)
 
     """On bootup we want to scan all directories and look for files that
     weren't there or files that changed before media-monitor process

python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py

@@ -2,6 +2,8 @@ import socket
 import logging
 import time
 import os
+import shutil
+import difflib
 
 import pyinotify
 from pyinotify import ProcessEvent
@@ -40,6 +42,8 @@ class AirtimeProcessEvent(ProcessEvent):
         self.create_dict = {}
         self.mount_file_dir = "/etc";
         self.mount_file = "/etc/mtab";
+        self.curr_mtab_file = "/var/tmp/airtime/media-monitor/currMtab"
+        self.prev_mtab_file = "/var/tmp/airtime/media-monitor/prevMtab"
 
     def add_filepath_to_ignore(self, filepath):
         self.ignore_event.add(filepath)
@@ -167,23 +171,33 @@ class AirtimeProcessEvent(ProcessEvent):
     # if change is detected on /etc/mtab, we check what mount(file system) was added/removed
     # and act accordingly
     def handle_mount_change(self):
-        mount_list = [];
-        # parse /etc/mtab
-        fh = open(self.mount_file, 'r')
-        while 1:
-            line = fh.readline()
-            if not line:
-                break
-            
-            line_info = line.split(' ')
-            # the line format is like following:
-            # /dev/sdg1 /media/809D-D2A1 vfat rw,nosuid,nodev,uhelper=udisks..........
-            # so we always get [1] after split to get the mount point
-            mount_list.append(line_info[1])
-        fh.close()
-        self.logger.info("Mount List: %s", mount_list)
+        self.logger.info("Mount change detected, handling changes...");
+        # take snapshot of mtab file and update currMtab and prevMtab
+        # move currMtab to prevMtab and create new currMtab
+        shutil.move(self.curr_mtab_file, self.prev_mtab_file)
+        # create the file
+        shutil.copy(self.mount_file, self.curr_mtab_file)
+        
+        d = difflib.Differ()
+        curr_fh = open(self.curr_mtab_file, 'r')
+        prev_fh = open(self.prev_mtab_file, 'r')
+        
+        diff = list(d.compare(prev_fh.readlines(), curr_fh.readlines()))
+        added_mount_points = []
+        removed_mount_points = []
+        
+        for dir in diff:
+            info = dir.split(' ')
+            if info[0] == '+':
+                added_mount_points.append(info[2])
+            elif info[0] == '-':
+                removed_mount_points.append(info[2])
+        
+        self.logger.info("added: %s", added_mount_points)
+        self.logger.info("removed: %s", removed_mount_points)
+        
         # send current mount information to Airtime
-        self.api_client.update_file_system_mount(mount_list);
+        self.api_client.update_file_system_mount(added_mount_points, removed_mount_points);
     
     def handle_watched_dir_missing(self, dir):
         self.api_client.handle_watched_dir_missing(dir);

python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py

@@ -10,7 +10,7 @@ import pyinotify
 
 class MediaMonitorCommon:
 
-    timestamp_file = "/var/tmp/airtime/last_index"
+    timestamp_file = "/var/tmp/airtime/media-monitor/last_index"
 
     def __init__(self, airtime_config, wm=None):
         self.supported_file_formats = ['mp3', 'ogg']

python_apps/media-monitor/install/media-monitor-initialize.py

@@ -6,7 +6,9 @@ if os.geteuid() != 0:
     sys.exit(1)
 
 try:
-    
+    #create media-monitor dir under /var/tmp/airtime
+    if not os.path.exists("/var/tmp/airtime/media-monitor"):
+        os.makedirs("/var/tmp/airtime/media-monitor")
     if os.environ["disable_auto_start_services"] == "f":
         #update-rc.d init script
         p = Popen("update-rc.d airtime-media-monitor defaults >/dev/null 2>&1", shell=True)