diff --git a/python_apps/media-monitor2/media/monitor/events.py b/python_apps/media-monitor2/media/monitor/events.py index 2781dfde4..841aebbfc 100644 --- a/python_apps/media-monitor2/media/monitor/events.py +++ b/python_apps/media-monitor2/media/monitor/events.py @@ -2,8 +2,10 @@ import os import abc from media.monitor.pure import LazyProperty +import media.monitor.pure as mmp from media.monitor.metadata import Metadata from media.monitor.log import Loggable +from media.monitor.exceptions import BadSongFile class PathChannel(object): """a dumb struct; python has no record types""" @@ -56,13 +58,26 @@ class BaseEvent(Loggable): def __str__(self): return "Event. Path: %s" % self.__raw_event.pathname + def is_dir_event(self): + return self._raw_event.dir + # nothing to see here, please move along def morph_into(self, evt): + """ + 'Morphing' should preserve the self.cookie invariant. I.e. either + None -> None or int -> int + """ self.logger.info("Morphing '%s' into '%s'" % (self.__class__.__name__, evt.__class__.__name__)) self._raw_event = evt self.path = evt.path self.__class__ = evt.__class__ + return self + +class FakePyinotify(object): + def __init__(self, path): + self.pathname = path + class OrganizeFile(BaseEvent, HasMetaData): def __init__(self, *args, **kwargs): super(OrganizeFile, self).__init__(*args, **kwargs) @@ -78,7 +93,7 @@ class NewFile(BaseEvent, HasMetaData): req_dict = self.metadata.extract() req_dict['mode'] = u'create' req_dict['MDATA_KEY_FILEPATH'] = unicode( self.path ) - return req_dict + return [req_dict] class DeleteFile(BaseEvent): def __init__(self, *args, **kwargs): super(DeleteFile, self).__init__(*args, **kwargs) @@ -86,7 +101,7 @@ class DeleteFile(BaseEvent): req_dict = {} req_dict['mode'] = u'delete' req_dict['MDATA_KEY_FILEPATH'] = unicode( self.path ) - return req_dict + return [req_dict] class MoveFile(BaseEvent, HasMetaData): """Path argument should be the new path of the file that was moved""" @@ -96,15 +111,34 @@ class MoveFile(BaseEvent, HasMetaData): req_dict['mode'] = u'moved' req_dict['MDATA_KEY_MD5'] = self.metadata.extract()['MDATA_KEY_MD5'] req_dict['MDATA_KEY_FILEPATH'] = unicode( self.path ) - return req_dict + return [req_dict] + +def map_events(directory, constructor): + #return map(lambda f: constructor( FakePyinotify(f) ).pack(), + #mmp.walk_supported(directory.replace("-unknown-path",""), + #clean_empties=False)) + for f in mmp.walk_supported(directory.replace("-unknown-path",""), + clean_empties=True): + try: yield constructor( FakePyinotify(f) ).pack()[0] + except BadSongFile as e: yield e class DeleteDir(BaseEvent): def __init__(self, *args, **kwargs): super(DeleteDir, self).__init__(*args, **kwargs) + def pack(self): + return map_events( self.path, DeleteFile ) + +class MoveDir(BaseEvent): + def __init__(self, *args, **kwargs): super(MoveDir, self).__init__(*args, **kwargs) + def pack(self): + return map_events( self.path, MoveFile ) + +class DeleteDirWatch(BaseEvent): + def __init__(self, *args, **kwargs): super(DeleteDirWatch, self).__init__(*args, **kwargs) def pack(self): req_dict = {} req_dict['mode'] = u'delete_dir' req_dict['MDATA_KEY_FILEPATH'] = unicode( self.path ) - return req_dict + return [req_dict] class ModifyFile(BaseEvent, HasMetaData): def __init__(self, *args, **kwargs): super(ModifyFile, self).__init__(*args, **kwargs) @@ -113,5 +147,5 @@ class ModifyFile(BaseEvent, HasMetaData): req_dict['mode'] = u'modify' # path to directory that is to be removed req_dict['MDATA_KEY_FILEPATH'] = unicode( self.path ) - return req_dict + return [req_dict] diff --git a/python_apps/media-monitor2/media/monitor/listeners.py b/python_apps/media-monitor2/media/monitor/listeners.py index b0dd39aee..28d3c874f 100644 --- a/python_apps/media-monitor2/media/monitor/listeners.py +++ b/python_apps/media-monitor2/media/monitor/listeners.py @@ -5,7 +5,8 @@ from pydispatch import dispatcher import media.monitor.pure as mmp from media.monitor.pure import IncludeOnly from media.monitor.events import OrganizeFile, NewFile, MoveFile, DeleteFile, \ - ModifyFile, DeleteDir, EventRegistry + ModifyFile, DeleteDir, EventRegistry, MoveDir,\ + DeleteDirWatch from media.monitor.log import Loggable, get_logger # We attempt to document a list of all special cases and hacks that the @@ -103,35 +104,47 @@ class OrganizeListener(BaseListener, pyinotify.ProcessEvent, Loggable): dispatcher.send(signal=self.signal, sender=self, event=OrganizeFile(event)) class StoreWatchListener(BaseListener, Loggable, pyinotify.ProcessEvent): - + # TODO : must intercept DeleteDirWatch events somehow def process_IN_CLOSE_WRITE(self, event): self.process_create(event) def process_IN_MOVED_TO(self, event): if EventRegistry.registered(event): - EventRegistry.matching(event).morph_into(MoveFile(event)) + # We need this trick because we don't how to "expand" dir events + # into file events until we know for sure if we deleted or moved + morph = MoveDir(event) if event.dir else MoveFile(event) + EventRegistry.matching(event).morph_into(morph) else: self.process_create(event) def process_IN_MOVED_FROM(self, event): # Is either delete dir or delete file - evt = self.process_delete_dir(event) if event.dir else self.process_delete(event) + evt = self.process_delete(event) if hasattr(event,'cookie'): EventRegistry.register(evt) def process_IN_DELETE(self,event): self.process_delete(event) # Capturing modify events is too brittle and error prone # def process_IN_MODIFY(self,event): self.process_modify(event) + # TODO : Remove this code. Later decided we will ignore modify events + # since it's too difficult to tell which ones should be handled. Much + # easier to just intercept IN_CLOSE_WRITE and decide what to do on the php + # side @mediate_ignored @IncludeOnly(mmp.supported_extensions) def process_modify(self, event): FileMediator.skip_next('IN_MODIFY','IN_CLOSE_WRITE',key='maskname') - dispatcher.send(signal=self.signal, sender=self, event=ModifyFile(event)) + evt = ModifyFile(event) + dispatcher.send(signal=self.signal, sender=self, event=evt) @mediate_ignored @IncludeOnly(mmp.supported_extensions) def process_create(self, event): - dispatcher.send(signal=self.signal, sender=self, event=NewFile(event)) + evt = NewFile(event) + dispatcher.send(signal=self.signal, sender=self, event=evt) + return evt @mediate_ignored @IncludeOnly(mmp.supported_extensions) def process_delete(self, event): - evt = DeleteFile(event) + evt = None + if event.dir: evt = DeleteDir(event) + else: evt = DeleteFile(event) dispatcher.send(signal=self.signal, sender=self, event=evt) return evt diff --git a/python_apps/media-monitor2/media/monitor/pure.py b/python_apps/media-monitor2/media/monitor/pure.py index e987022eb..61ea635ee 100644 --- a/python_apps/media-monitor2/media/monitor/pure.py +++ b/python_apps/media-monitor2/media/monitor/pure.py @@ -42,7 +42,9 @@ class IncludeOnly(object): def __call__(self, func): def _wrap(moi, event, *args, **kwargs): ext = extension(event.pathname) - if ext in self.exts: return func(moi, event, *args, **kwargs) + # Checking for emptiness b/c we don't want to skip direcotries + if (ext in self.exts) or event.dir: + return func(moi, event, *args, **kwargs) return _wrap diff --git a/python_apps/media-monitor2/media/monitor/watchersyncer.py b/python_apps/media-monitor2/media/monitor/watchersyncer.py index 0baab9654..31c662c69 100644 --- a/python_apps/media-monitor2/media/monitor/watchersyncer.py +++ b/python_apps/media-monitor2/media/monitor/watchersyncer.py @@ -33,8 +33,12 @@ class RequestSync(threading.Thread,Loggable): # A simplistic request would like: # TODO : recorded shows aren't flagged right packed_requests = [] - for request in self.requests: - try: packed_requests.append(request.pack()) + for request_event in self.requests: + try: + for request in request_event.pack(): + if isinstance(request, BadSongFile): + self.logger.info("Bad song file: '%s'" % request.path) + else: packed_requests.append(request) except BadSongFile as e: self.logger.info("Bad song file: '%s'" % e.path) self.logger.info("TODO : put in ignore list") @@ -53,7 +57,7 @@ class RequestSync(threading.Thread,Loggable): self.logger.info("Trying again after %f seconds" % self.request_wait) time.sleep( self.request_wait ) except Exception as e: - self.logger.unexpected_exception(e) + self.unexpected_exception(e) else: self.logger.info("Request worked on the '%d' try" % (try_index + 1)) break