From f7b48a4dbb5a06c93fb8155c7823433ed601d6c9 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Wed, 1 Aug 2012 17:46:47 -0400 Subject: [PATCH] cc-4105: fixed some bugs. fixed manager tests --- python_apps/media-monitor2/media/monitor/airtime.py | 8 ++++++-- .../media-monitor2/media/monitor/bootstrap.py | 13 +++++++++---- python_apps/media-monitor2/media/monitor/handler.py | 1 + .../media-monitor2/media/monitor/listeners.py | 1 - python_apps/media-monitor2/media/monitor/manager.py | 8 +------- python_apps/media-monitor2/media/monitor/pure.py | 6 ++++++ python_apps/media-monitor2/media/monitor/syncdb.py | 4 ++-- .../media-monitor2/media/monitor/watchersyncer.py | 11 ++++++++++- python_apps/media-monitor2/tests/test_manager.py | 11 ++--------- 9 files changed, 37 insertions(+), 26 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/airtime.py b/python_apps/media-monitor2/media/monitor/airtime.py index b6bf5b440..ba4e83b20 100644 --- a/python_apps/media-monitor2/media/monitor/airtime.py +++ b/python_apps/media-monitor2/media/monitor/airtime.py @@ -5,6 +5,7 @@ import json import os import time import copy +import traceback from media.monitor.exceptions import BadSongFile from media.monitor.metadata import Metadata @@ -95,6 +96,7 @@ class AirtimeMessageReceiver(Loggable): if (not directory_id) and (not directory): raise ValueError("You must provide either directory_id or directory") sdb = AirtimeDB(apc.AirtimeApiClient.create_right_config()) + if directory: directory = os.path.normpath(directory) if directory_id == None: directory_id = sdb.to_id(directory) if directory == None: directory = sdb.to_directory(directory_id) try: @@ -103,6 +105,7 @@ class AirtimeMessageReceiver(Loggable): except Exception as e: self.logger.info( "Exception bootstrapping: (dir,id)=(%s,%s)" % (directory, directory_id) ) self.logger.info( str(e) ) + self.logger.error( traceback.format_exc() ) raise DirectoryIsNotListed(directory) def supported_messages(self): @@ -126,11 +129,12 @@ class AirtimeMessageReceiver(Loggable): except Exception as e: self.logger.info("Failed to create watched dir '%s'" % msg['directory']) self.logger.info(str(e)) - # Is this clever or stupid? else: self.new_watch(msg) else: # TODO : Refactor this; breaks encapsulation. - self.manager.watch_listener.flush_events(msg['directory']) + # TODO : Should not flush events, should be smarted and bootstrap + #self.manager.watch_listener.flush_events(msg['directory']) + self.__request_now_bootstrap( directory=msg['directory'] ) self.manager.add_watch_directory(msg['directory']) def remove_watch(self, msg): diff --git a/python_apps/media-monitor2/media/monitor/bootstrap.py b/python_apps/media-monitor2/media/monitor/bootstrap.py index 29b3c4ca4..1892a65f1 100644 --- a/python_apps/media-monitor2/media/monitor/bootstrap.py +++ b/python_apps/media-monitor2/media/monitor/bootstrap.py @@ -33,7 +33,7 @@ class Bootstrapper(Loggable): or add a watched/imported directory """ songs = set([]) - modded = deleted = 0 + added = modded = deleted = 0 for f in mmp.walk_supported(directory, clean_empties=False): songs.add(f) # We decide whether to update a file's metadata by checking @@ -46,13 +46,18 @@ class Bootstrapper(Loggable): modded += 1 dispatcher.send(signal=self.watch_signal, sender=self, event=DeleteFile(f)) dispatcher.send(signal=self.watch_signal, sender=self, event=NewFile(f)) - db_songs = self.db.directory_get_files(directory) + db_songs = set(( song for song in self.db.directory_get_files(directory) if mmp.sub_path(directory,song) )) # Get all the files that are in the database but in the file # system. These are the files marked for deletions for to_delete in db_songs.difference(songs): dispatcher.send(signal=self.watch_signal, sender=self, event=DeleteFile(to_delete)) deleted += 1 - self.logger.info( "Flushed watch directory(%s). (modified, deleted) = (%d, %d)" - % (directory, modded, deleted) ) + for to_add in songs.difference(db_songs): + if len(songs.difference(db_songs)) > 100: + import ipdb; ipdb.set_trace() + dispatcher.send(signal=self.watch_signal, sender=self, event=NewFile(to_add)) + added += 1 + self.logger.info( "Flushed watch directory (%s). (added, modified, deleted) = (%d, %d, %d)" + % (directory, added, modded, deleted) ) diff --git a/python_apps/media-monitor2/media/monitor/handler.py b/python_apps/media-monitor2/media/monitor/handler.py index 737493f68..c9b6b5931 100644 --- a/python_apps/media-monitor2/media/monitor/handler.py +++ b/python_apps/media-monitor2/media/monitor/handler.py @@ -39,6 +39,7 @@ class ProblemFileHandler(Handles, Loggable): def handle(self, sender, event, exception=None): self.logger.info("Received problem file: '%s'. Supposed to move it to problem dir", event.path) + import ipdb; ipdb.set_trace() try: mmp.move_to_dir(dir_path=self.problem_dir, file_path=event.path) except Exception as e: self.logger.info("Could not move file: '%s' to problem dir: '%s'" % (event.path, self.problem_dir)) diff --git a/python_apps/media-monitor2/media/monitor/listeners.py b/python_apps/media-monitor2/media/monitor/listeners.py index bee457bc0..61e89eac2 100644 --- a/python_apps/media-monitor2/media/monitor/listeners.py +++ b/python_apps/media-monitor2/media/monitor/listeners.py @@ -87,7 +87,6 @@ class StoreWatchListener(BaseListener, Loggable, pyinotify.ProcessEvent): def process_IN_DELETE(self,event): self.process_delete(event) def process_IN_MODIFY(self,event): self.process_modify(event) - @mediate_ignored @IncludeOnly(mmp.supported_extensions) def process_modify(self, event): diff --git a/python_apps/media-monitor2/media/monitor/manager.py b/python_apps/media-monitor2/media/monitor/manager.py index 2226c9983..27a1b2946 100644 --- a/python_apps/media-monitor2/media/monitor/manager.py +++ b/python_apps/media-monitor2/media/monitor/manager.py @@ -45,7 +45,7 @@ class Manager(Loggable): Manager.global_inst = self def watch_signal(self): - return self.watch_listener.self.signal + return self.watch_listener.signal def __remove_watch(self,path): if path in self.__wd_path: # only delete if dir is actually being watched @@ -111,11 +111,6 @@ class Manager(Loggable): self.__remove_watch(self.organize['imported_path']) self.organize['imported_path'] = new_path self.organize['organizer'] = self.__create_organizer(new_path) - # flush all the files in the new store_directory. this is done so that - # new files are added to the database. Note that we are not responsible - # for removing songs in the old store directory from the database - # we assume that this is already done for us. - # self.watch_listener.flush_events(new_path) self.__add_watch(new_path, self.watch_listener) def change_storage_root(self, store): @@ -165,7 +160,6 @@ class Manager(Loggable): def pyinotify(self): return pyinotify.Notifier(self.wm) - def loop(self): """ block until we receive pyinotify events diff --git a/python_apps/media-monitor2/media/monitor/pure.py b/python_apps/media-monitor2/media/monitor/pure.py index dc946bf72..f89d2d38a 100644 --- a/python_apps/media-monitor2/media/monitor/pure.py +++ b/python_apps/media-monitor2/media/monitor/pure.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import copy import os +from os.path import normpath import shutil import sys import hashlib @@ -328,6 +329,11 @@ def create_dir(path): else: # if no error occurs we still need to check that dir exists if not os.path.exists: raise FailedToCreateDir(path) +def sub_path(directory,f): + normalized = normpath(directory) + common = os.path.commonprefix([ directory, normpath(f) ]) + return common == normalized + if __name__ == '__main__': import doctest doctest.testmod() diff --git a/python_apps/media-monitor2/media/monitor/syncdb.py b/python_apps/media-monitor2/media/monitor/syncdb.py index 9a232e287..3b65e50a1 100644 --- a/python_apps/media-monitor2/media/monitor/syncdb.py +++ b/python_apps/media-monitor2/media/monitor/syncdb.py @@ -83,9 +83,9 @@ class AirtimeDB(Loggable): raise NoDirectoryInAirtime( normal_dir, self.dir_to_id ) all_files = self.dir_id_get_files( self.dir_to_id[normal_dir] ) if normal_dir == self.recorded_path(): - all_files = [ p for p in all_files if len(os.path.commonprefix([ self.recorded_path(), p ])) > 0 ] + all_files = [ p for p in all_files if mmp.sub_path( self.recorded_path(), p ) ] elif normal_dir == self.import_path(): - all_files = [ p for p in all_files if len(os.path.commonprefix([ self.import_path(), p ])) > 0 ] + all_files = [ p for p in all_files if mmp.sub_path( self.import_path(), p ) ] elif normal_dir == self.storage_path(): self.logger.info("Warning, you're getting all files in '%s' which includes imported + record" % normal_dir) diff --git a/python_apps/media-monitor2/media/monitor/watchersyncer.py b/python_apps/media-monitor2/media/monitor/watchersyncer.py index 45b1a42d7..8bd13ab9d 100644 --- a/python_apps/media-monitor2/media/monitor/watchersyncer.py +++ b/python_apps/media-monitor2/media/monitor/watchersyncer.py @@ -2,6 +2,7 @@ import threading import time import copy +import traceback from media.monitor.handler import ReportHandler from media.monitor.events import NewFile, DeleteFile @@ -32,7 +33,15 @@ class RequestSync(threading.Thread,Loggable): # to recorded shows # A simplistic request would like: # TODO : recorded shows aren't flagged right - packed_requests = [ req.pack() for req in self.requests ] + packed_requests = [] + for request in self.requests: + try: packed_requests.append(request.pack()) + except BadSongFile as e: + self.logger.info("Bad song file: '%s'" % e.path) + self.logger.info("TODO : put in ignore list") + except Exception as e: + self.logger.info("An evil exception occured to packing '%s'" % request.path) + self.logger.error( traceback.format_exc() ) # Remove when finished debugging def send_one(x): self.apiclient.send_media_monitor_requests( [x] ) def make_req(): self.apiclient.send_media_monitor_requests( packed_requests ) diff --git a/python_apps/media-monitor2/tests/test_manager.py b/python_apps/media-monitor2/tests/test_manager.py index a09eeb2e7..abd6e9e84 100644 --- a/python_apps/media-monitor2/tests/test_manager.py +++ b/python_apps/media-monitor2/tests/test_manager.py @@ -20,16 +20,9 @@ class TestManager(unittest.TestCase): def test_organize_path(self): man = Manager() man.set_organize_path( self.opath ) - self.assertEqual( man.organize_path, self.opath ) + self.assertEqual( man.get_organize_path(), self.opath ) man.set_organize_path( self.ppath ) - self.assertEqual( man.organize_path, self.ppath ) - - def test_store_path(self): - man = Manager() - man.set_store_path( self.opath ) - self.assertEqual( man.store_path, self.opath ) - man.set_store_path( self.ppath ) - self.assertEqual( man.store_path, self.ppath ) + self.assertEqual( man.get_organize_path(), self.ppath ) def test_add_watch_directory(self): man = Manager()