cc-4105: fixed some bugs. fixed manager tests

python_apps/media-monitor2/media/monitor/airtime.py

@@ -5,6 +5,7 @@ import json
 import os
 import time
 import copy
+import traceback
 
 from media.monitor.exceptions import BadSongFile
 from media.monitor.metadata import Metadata
@@ -95,6 +96,7 @@ class AirtimeMessageReceiver(Loggable):
         if (not directory_id) and (not directory):
             raise ValueError("You must provide either directory_id or directory")
         sdb = AirtimeDB(apc.AirtimeApiClient.create_right_config())
+        if directory: directory = os.path.normpath(directory)
         if directory_id == None: directory_id = sdb.to_id(directory)
         if directory == None: directory = sdb.to_directory(directory_id)
         try:
@@ -103,6 +105,7 @@ class AirtimeMessageReceiver(Loggable):
         except Exception as e:
             self.logger.info( "Exception bootstrapping: (dir,id)=(%s,%s)" % (directory, directory_id) )
             self.logger.info( str(e) )
+            self.logger.error( traceback.format_exc() )
             raise DirectoryIsNotListed(directory)
 
     def supported_messages(self):
@@ -126,11 +129,12 @@ class AirtimeMessageReceiver(Loggable):
             except Exception as e:
                 self.logger.info("Failed to create watched dir '%s'" % msg['directory'])
                 self.logger.info(str(e))
-            # Is this clever or stupid?
             else: self.new_watch(msg)
         else:
             # TODO : Refactor this; breaks encapsulation.
-            self.manager.watch_listener.flush_events(msg['directory'])
+            # TODO : Should not flush events, should be smarted and bootstrap
+            #self.manager.watch_listener.flush_events(msg['directory'])
+            self.__request_now_bootstrap( directory=msg['directory'] )
             self.manager.add_watch_directory(msg['directory'])
 
     def remove_watch(self, msg):

python_apps/media-monitor2/media/monitor/bootstrap.py

@@ -33,7 +33,7 @@ class Bootstrapper(Loggable):
         or add a watched/imported directory
         """
         songs = set([])
-        modded = deleted = 0
+        added = modded = deleted = 0
         for f in mmp.walk_supported(directory, clean_empties=False):
             songs.add(f)
             # We decide whether to update a file's metadata by checking
@@ -46,13 +46,18 @@ class Bootstrapper(Loggable):
                 modded += 1
                 dispatcher.send(signal=self.watch_signal, sender=self, event=DeleteFile(f))
                 dispatcher.send(signal=self.watch_signal, sender=self, event=NewFile(f))
-        db_songs = self.db.directory_get_files(directory)
+        db_songs = set(( song for song in self.db.directory_get_files(directory) if mmp.sub_path(directory,song) ))
         # Get all the files that are in the database but in the file
         # system. These are the files marked for deletions
         for to_delete in db_songs.difference(songs):
             dispatcher.send(signal=self.watch_signal, sender=self, event=DeleteFile(to_delete))
             deleted += 1
-        self.logger.info( "Flushed watch directory(%s). (modified, deleted) = (%d, %d)"
-                        % (directory, modded, deleted) )
+        for to_add in songs.difference(db_songs):
+            if len(songs.difference(db_songs)) > 100:
+                import ipdb; ipdb.set_trace()
+            dispatcher.send(signal=self.watch_signal, sender=self, event=NewFile(to_add))
+            added += 1
+        self.logger.info( "Flushed watch directory (%s). (added, modified, deleted) = (%d, %d, %d)"
+                        % (directory, added, modded, deleted) )
 
 

python_apps/media-monitor2/media/monitor/handler.py

@@ -39,6 +39,7 @@ class ProblemFileHandler(Handles, Loggable):
 
     def handle(self, sender, event, exception=None):
         self.logger.info("Received problem file: '%s'. Supposed to move it to problem dir", event.path)
+        import ipdb; ipdb.set_trace()
         try: mmp.move_to_dir(dir_path=self.problem_dir, file_path=event.path)
         except Exception as e:
             self.logger.info("Could not move file: '%s' to problem dir: '%s'" % (event.path, self.problem_dir))

python_apps/media-monitor2/media/monitor/listeners.py

@@ -87,7 +87,6 @@ class StoreWatchListener(BaseListener, Loggable, pyinotify.ProcessEvent):
     def process_IN_DELETE(self,event): self.process_delete(event)
     def process_IN_MODIFY(self,event): self.process_modify(event)
 
-
     @mediate_ignored
     @IncludeOnly(mmp.supported_extensions)
     def process_modify(self, event):

python_apps/media-monitor2/media/monitor/manager.py

@@ -45,7 +45,7 @@ class Manager(Loggable):
         Manager.global_inst = self
 
     def watch_signal(self):
-        return self.watch_listener.self.signal
+        return self.watch_listener.signal
 
     def __remove_watch(self,path):
         if path in self.__wd_path: # only delete if dir is actually being watched
@@ -111,11 +111,6 @@ class Manager(Loggable):
         self.__remove_watch(self.organize['imported_path'])
         self.organize['imported_path'] = new_path
         self.organize['organizer'] = self.__create_organizer(new_path)
-        # flush all the files in the new store_directory. this is done so that
-        # new files are added to the database. Note that we are not responsible
-        # for removing songs in the old store directory from the database
-        # we assume that this is already done for us.
-        # self.watch_listener.flush_events(new_path)
         self.__add_watch(new_path, self.watch_listener)
 
     def change_storage_root(self, store):
@@ -165,7 +160,6 @@ class Manager(Loggable):
     def pyinotify(self):
         return pyinotify.Notifier(self.wm)
 
-
     def loop(self):
         """
         block until we receive pyinotify events

python_apps/media-monitor2/media/monitor/pure.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 import copy
 import os
+from os.path import normpath
 import shutil
 import sys
 import hashlib
@@ -328,6 +329,11 @@ def create_dir(path):
         else: # if no error occurs we still need to check that dir exists
             if not os.path.exists: raise FailedToCreateDir(path)
 
+def sub_path(directory,f):
+    normalized = normpath(directory)
+    common = os.path.commonprefix([ directory, normpath(f) ])
+    return common == normalized
+
 if __name__ == '__main__':
     import doctest
     doctest.testmod()

python_apps/media-monitor2/media/monitor/syncdb.py

@@ -83,9 +83,9 @@ class AirtimeDB(Loggable):
             raise NoDirectoryInAirtime( normal_dir, self.dir_to_id )
         all_files = self.dir_id_get_files( self.dir_to_id[normal_dir] )
         if normal_dir == self.recorded_path():
-            all_files = [ p for p in all_files if len(os.path.commonprefix([ self.recorded_path(), p ])) > 0 ]
+            all_files = [ p for p in all_files if mmp.sub_path( self.recorded_path(), p ) ]
         elif normal_dir == self.import_path():
-            all_files = [ p for p in all_files if len(os.path.commonprefix([ self.import_path(), p ])) > 0 ]
+            all_files = [ p for p in all_files if mmp.sub_path( self.import_path(), p ) ]
         elif normal_dir == self.storage_path():
             self.logger.info("Warning, you're getting all files in '%s' which includes imported + record"
                     % normal_dir)

python_apps/media-monitor2/media/monitor/watchersyncer.py

@@ -2,6 +2,7 @@
 import threading
 import time
 import copy
+import traceback
 
 from media.monitor.handler import ReportHandler
 from media.monitor.events import NewFile, DeleteFile
@@ -32,7 +33,15 @@ class RequestSync(threading.Thread,Loggable):
         # to recorded shows
         # A simplistic request would like:
         # TODO : recorded shows aren't flagged right
-        packed_requests = [ req.pack() for req in self.requests ]
+        packed_requests = []
+        for request in self.requests:
+            try: packed_requests.append(request.pack())
+            except BadSongFile as e:
+                self.logger.info("Bad song file: '%s'" % e.path)
+                self.logger.info("TODO : put in ignore list")
+            except Exception as e:
+                self.logger.info("An evil exception occured to packing '%s'" % request.path)
+                self.logger.error( traceback.format_exc() )
         # Remove when finished debugging
         def send_one(x): self.apiclient.send_media_monitor_requests( [x] )
         def make_req(): self.apiclient.send_media_monitor_requests( packed_requests )

python_apps/media-monitor2/tests/test_manager.py

@@ -20,16 +20,9 @@ class TestManager(unittest.TestCase):
     def test_organize_path(self):
         man = Manager()
         man.set_organize_path( self.opath )
-        self.assertEqual( man.organize_path, self.opath )
+        self.assertEqual( man.get_organize_path(), self.opath )
         man.set_organize_path( self.ppath )
-        self.assertEqual( man.organize_path, self.ppath )
-
-    def test_store_path(self):
-        man = Manager()
-        man.set_store_path( self.opath )
-        self.assertEqual( man.store_path, self.opath )
-        man.set_store_path( self.ppath )
-        self.assertEqual( man.store_path, self.ppath )
+        self.assertEqual( man.get_organize_path(), self.ppath )
 
     def test_add_watch_directory(self):
         man = Manager()