CC-2419: Media monitor does not import files that already existed in /srv/airtime/stor

-work in progress

python_apps/media-monitor/MediaMonitor.py

@@ -6,7 +6,7 @@ import sys
 import os
 import signal
 
-from multiprocessing import Process
+from multiprocessing import Process, Queue as mpQueue
 
 from airtimefilemonitor.airtimenotifier import AirtimeNotifier
 from airtimefilemonitor.airtimeprocessevent import AirtimeProcessEvent
@@ -36,18 +36,20 @@ processes = []
 try:
     config = AirtimeMediaConfig(logger)
     
-    bootstrap = AirtimeMediaMonitorBootstrap(logger)
+    multi_queue = mpQueue()
-    bootstrap.scan()
     
+    bootstrap = AirtimeMediaMonitorBootstrap(logger, multi_queue)
+    bootstrap.scan()
+        
     logger.info("Initializing event processor")
-    pe = AirtimeProcessEvent(airtime_config=config)
+    pe = AirtimeProcessEvent(multi_queue, airtime_config=config)
 
     notifier = AirtimeNotifier(pe.wm, pe, read_freq=0.1, timeout=0.1, airtime_config=config)
     notifier.coalesce_events()
 
     #create 5 worker processes
     for i in range(5):
-        p = Process(target=notifier.process_file_events, args=(pe.multi_queue,))
+        p = Process(target=notifier.process_file_events, args=(multi_queue,))
         processes.append(p)
         p.start()
 

python_apps/media-monitor/airtimefilemonitor/airtimemediamonitorbootstrap.py

@@ -5,8 +5,9 @@ from subprocess import Popen, PIPE
 
 class AirtimeMediaMonitorBootstrap():
 
-    def __init__(self, logger):
+    def __init__(self, logger, multi_queue):
         self.logger = logger
+        self.multi_queue = multi_queue
         
     """
     on bootup we want to scan all directories and look for files that
@@ -21,21 +22,21 @@ class AirtimeMediaMonitorBootstrap():
             
     def check_for_diff(self, dir):
         airtime_tmp = '/var/tmp/airtime'
+
+        #find files that have been modified since the last time
+        #media-monitor process was running.
+        command = "find %s -type f -iname '*.ogg' -o -iname '*.mp3' -readable -mmin -30" % dir
+        stdout = self.execCommandAndReturnStdOut(command)
+        self.logger.info("Files modified since last checkin: \n%s\n", stdout)
+
+        #TODO: notify about modified and newly created files (not including copied files)
     
         if os.path.exists(airtime_tmp + '/.airtime_media_index'):
             #a previous index exists, we can do a diff between this
             #file and the current state to see whether anything has
             #changed.
             self.logger.info("Previous index file found.")
-            
+                        
-                
-            #find files that have been modified since the last time
-            #media-monitor process was running.
-            command = "find %s -type f -iname '*.ogg' -o -iname '*.mp3' -readable -mmin -30" % dir
-            stdout = self.execCommandAndReturnStdOut(command)
-            self.logger.info("Files modified since last checkin: \n%s\n", stdout)
-            
-            
             #find deleted files
             command = "find %s -type f -iname '*.ogg' -o -iname '*.mp3' -readable > %s/.airtime_media_index.tmp" % (dir, airtime_tmp)
             self.execCommand(command)
@@ -43,6 +44,8 @@ class AirtimeMediaMonitorBootstrap():
             command = "diff %s/.airtime_media_index.tmp %s/.airtime_media_index" % (airtime_tmp, airtime_tmp)
             stdout = self.execCommandAndReturnStdOut(command)
             self.logger.info("Deleted files since last checkin:\n%s\n", stdout)
+
+            #TODO: notify about deleted files and files moved here
             
         else:
             #a previous index does not exist. Most likely means that 
@@ -53,6 +56,9 @@ class AirtimeMediaMonitorBootstrap():
             #create a new index file.
             command = "find %s -type f -iname '*.ogg' -o -iname '*.mp3' -readable > %s/.airtime_media_index" % (dir, airtime_tmp)
             self.execCommand(command)
+
+            #TODO: notify about all files in this directory.
+            self.multi_queue.put(event)
                 
     def execCommand(self, command):
         p = Popen(command, shell=True)
@@ -71,4 +77,4 @@ class AirtimeMediaMonitorBootstrap():
 if __name__ == '__main__':
     
         mmb = AirtimeMediaMonitorBootstrap()
-        mmb.scan()
+        mmb.scan()

python_apps/media-monitor/airtimefilemonitor/airtimenotifier.py

@@ -104,6 +104,13 @@ class AirtimeNotifier(Notifier):
             mm.watch_directory(new_storage_directory)
 
 
+    #update airtime with information about files discovered in our
+    #watched directories. Pass in a dict() object with the following 
+    #attributes:
+    # -filepath
+    # -mode
+    # -data
+    # -is_recorded_show
     def update_airtime(self, d):
 
         filepath = d['filepath']

python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py

@@ -4,8 +4,6 @@ import grp
 import pwd
 import logging
 
-from multiprocessing import Process, Lock, Queue as mpQueue
-
 import pyinotify
 from pyinotify import WatchManager, Notifier, ProcessEvent
 
@@ -18,7 +16,7 @@ from airtimefilemonitor.mediaconfig import AirtimeMediaConfig
 
 class AirtimeProcessEvent(ProcessEvent):
 
-    def my_init(self, airtime_config=None):
+    def my_init(self, queue, airtime_config=None):
         """
         Method automatically called from ProcessEvent.__init__(). Additional
         keyworded arguments passed to ProcessEvent.__init__() are then
@@ -34,11 +32,12 @@ class AirtimeProcessEvent(ProcessEvent):
         self.gui_replaced = {}
         self.renamed_files = {}
         self.file_events = []
-        self.multi_queue = mpQueue()
+        self.multi_queue = queue
         self.mask = pyinotify.ALL_EVENTS
         self.wm = WatchManager()
         self.md_manager = AirtimeMetadata()
 
+    #define which directories the pyinotify WatchManager should watch.
     def watch_directory(self, directory):
         return self.wm.add_watch(directory, self.mask, rec=True, auto_add=True)
 
@@ -206,13 +205,8 @@ class AirtimeProcessEvent(ProcessEvent):
             elif self.is_audio_file(event.pathname):
                 if self.is_parent_directory(event.pathname, storage_directory):
                     self.set_needed_file_permissions(event.pathname, event.dir)
-                    file_md = self.md_manager.get_md_from_file(event.pathname)
+                    
-
+                    self.process_new_file(event.pathname)
-                    if file_md is not None:
-                        filepath, is_recorded_show = self.create_file_path(event.pathname, file_md)
-                        self.move_file(event.pathname, filepath)
-                        self.renamed_files[event.pathname] = filepath
-                        self.file_events.append({'mode': self.config.MODE_CREATE, 'filepath': filepath, 'data': file_md, 'is_recorded_show': is_recorded_show})
                 else:
                     self.file_events.append({'mode': self.config.MODE_CREATE, 'filepath': event.pathname, 'is_recorded_show': False})
 
@@ -220,7 +214,16 @@ class AirtimeProcessEvent(ProcessEvent):
             if self.is_parent_directory(event.pathname, storage_directory):
                 self.set_needed_file_permissions(event.pathname, event.dir)
 
+    def process_new_file(pathname):
+        file_md = self.md_manager.get_md_from_file(pathname)
 
+        if file_md is not None:
+            filepath, is_recorded_show = self.create_file_path(pathname, file_md)
+            self.move_file(pathname, filepath)
+            self.renamed_files[pathname] = filepath
+            self.file_events.append({'mode': self.config.MODE_CREATE, 'filepath': filepath, 'data': file_md, 'is_recorded_show': is_recorded_show})
+                
+                
     def process_IN_MODIFY(self, event):
         if not event.dir:
             self.logger.info("%s: %s", event.maskname, event.pathname)