From ba034479464e8ae1c3db8b784d804907606392ee Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Mon, 30 Jul 2012 16:50:48 -0400 Subject: [PATCH] cc-4105: added handling for pyinotify events and create more logical events --- .../application/controllers/ApiController.php | 6 ++--- .../airtimefilemonitor/airtimeprocessevent.py | 2 +- .../media-monitor2/media/monitor/airtime.py | 1 + .../media-monitor2/media/monitor/events.py | 27 +++++++++++++++++++ .../media/monitor/exceptions.py | 6 +++++ .../media-monitor2/media/monitor/handler.py | 2 ++ .../media-monitor2/media/monitor/listeners.py | 10 +++++-- .../media-monitor2/media/monitor/pure.py | 13 ++++++++- .../media/monitor/watchersyncer.py | 2 +- python_apps/media-monitor2/mm2.py | 8 +++++- 10 files changed, 68 insertions(+), 9 deletions(-) diff --git a/airtime_mvc/application/controllers/ApiController.php b/airtime_mvc/application/controllers/ApiController.php index f5af87e5f..923b83ab0 100644 --- a/airtime_mvc/application/controllers/ApiController.php +++ b/airtime_mvc/application/controllers/ApiController.php @@ -561,9 +561,9 @@ class ApiController extends Zend_Controller_Action $info_json = json_decode($raw_json, $assoc=true); $recorded = $info_json["is_record"]; unset( $info_json["is_record"] ); - unset( $info_json["MDATA_KEY_DURATION"] ); - unset( $info_json["MDATA_KEY_SAMPLERATE"] ); - unset( $info_json["MDATA_KEY_BITRATE"] ); + //unset( $info_json["MDATA_KEY_DURATION"] ); + //unset( $info_json["MDATA_KEY_SAMPLERATE"] ); + //unset( $info_json["MDATA_KEY_BITRATE"] ); if( !array_key_exists('mode', $info_json) ) { // Log invalid requests Logging::log("Received bad request(key=$k), no 'mode' parameter. Bad request is:"); diff --git a/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py b/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py index 90b8a0151..a3b22324e 100644 --- a/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py +++ b/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py @@ -67,7 +67,7 @@ class AirtimeProcessEvent(ProcessEvent): res = self.api_client.remove_watched_dir(path) if(res is None): self.logger.info("Unable to connect to the Airtime server.") - # sucess + # success if(res['msg']['code'] == 0): self.logger.info("%s removed from watch folder list successfully.", path) else: diff --git a/python_apps/media-monitor2/media/monitor/airtime.py b/python_apps/media-monitor2/media/monitor/airtime.py index d2d9a1d05..2ffa652f8 100644 --- a/python_apps/media-monitor2/media/monitor/airtime.py +++ b/python_apps/media-monitor2/media/monitor/airtime.py @@ -194,3 +194,4 @@ class AirtimeMessageReceiver(Loggable): self.logger.info("No clippy confirmation, ignoring event. Out of curiousity we will print some details.") self.logger.info(msg) + diff --git a/python_apps/media-monitor2/media/monitor/events.py b/python_apps/media-monitor2/media/monitor/events.py index c0c1d84ad..07a5975dc 100644 --- a/python_apps/media-monitor2/media/monitor/events.py +++ b/python_apps/media-monitor2/media/monitor/events.py @@ -56,3 +56,30 @@ class DeleteFile(BaseEvent): req_dict['mode'] = u'delete' req_dict['MDATA_KEY_FILEPATH'] = unicode( self.path ) return req_dict + +class MoveFile(BaseEvent, HasMetaData): + """Path argument should be the new path of the file that was moved""" + def __init__(self, *args, **kwargs): super(MoveFile, self).__init__(*args, **kwargs) + def pack(self): + req_dict = {} + req_dict['mode'] = u'moved' + req_dict['MDATA_KEY_MD5'] = self.metadata.extract()['MDATA_KEY_MD5'] + req_dict['MDATA_KEY_FILEPATH'] = unicode( self.path ) + return req_dict + +class DeleteDir(BaseEvent): + def __init__(self, *args, **kwargs): super(DeleteDir, self).__init__(*args, **kwargs) + def pack(self): + req_dict = {} + req_dict['mode'] = u'delete_dir' + req_dict['MDATA_KEY_FILEPATH'] = unicode( self.path ) + return req_dict + +class ModifyFile(BaseEvent, HasMetaData): + def __init__(self, *args, **kwargs): super(ModifyFile, self).__init__(*args, **kwargs) + def pack(self): + req_dict = self.metadata.extract() + req_dict['mode'] = u'modify' + # path to directory that is to be removed + req_dict['MDATA_KEY_FILEPATH'] = unicode( self.path ) + return req_dict diff --git a/python_apps/media-monitor2/media/monitor/exceptions.py b/python_apps/media-monitor2/media/monitor/exceptions.py index 184d7158a..3d5d305f8 100644 --- a/python_apps/media-monitor2/media/monitor/exceptions.py +++ b/python_apps/media-monitor2/media/monitor/exceptions.py @@ -32,3 +32,9 @@ class DirectoryIsNotListed(Exception): def __init__(self,dir_id): self.dir_id = dir_id def __str__(self): return "%d was not listed as a directory in the database" % self.dir_id + +class FailedToCreateDir(Exception): + def __init__(self,path, parent): + self.path = path + self.parent = parent + def __str__(self): return "Failed to create path '%s'" % self.path diff --git a/python_apps/media-monitor2/media/monitor/handler.py b/python_apps/media-monitor2/media/monitor/handler.py index f70d48177..737493f68 100644 --- a/python_apps/media-monitor2/media/monitor/handler.py +++ b/python_apps/media-monitor2/media/monitor/handler.py @@ -34,6 +34,8 @@ class ProblemFileHandler(Handles, Loggable): self.problem_dir = self.channel.path def dummy(sender, event, exception): self.handle(sender, event, exception) dispatcher.connect(dummy, signal=self.signal, sender=dispatcher.Any, weak=False) + mmp.create_dir( self.problem_dir ) + self.logger.info("Initialized problem file handler. Problem dir: '%s'" % self.problem_dir) def handle(self, sender, event, exception=None): self.logger.info("Received problem file: '%s'. Supposed to move it to problem dir", event.path) diff --git a/python_apps/media-monitor2/media/monitor/listeners.py b/python_apps/media-monitor2/media/monitor/listeners.py index 620dfd8fe..bee457bc0 100644 --- a/python_apps/media-monitor2/media/monitor/listeners.py +++ b/python_apps/media-monitor2/media/monitor/listeners.py @@ -4,7 +4,7 @@ from pydispatch import dispatcher import media.monitor.pure as mmp from media.monitor.pure import IncludeOnly -from media.monitor.events import OrganizeFile, NewFile, DeleteFile +from media.monitor.events import OrganizeFile, NewFile, DeleteFile, ModifyFile from media.monitor.log import Loggable, get_logger # We attempt to document a list of all special cases and hacks that the @@ -85,6 +85,13 @@ class StoreWatchListener(BaseListener, Loggable, pyinotify.ProcessEvent): def process_IN_MOVED_TO(self, event): self.process_create(event) def process_IN_MOVED_FROM(self, event): self.process_delete(event) def process_IN_DELETE(self,event): self.process_delete(event) + def process_IN_MODIFY(self,event): self.process_modify(event) + + + @mediate_ignored + @IncludeOnly(mmp.supported_extensions) + def process_modify(self, event): + dispatcher.send(signal=self.signal, sender=self, event=ModifyFile(event)) @mediate_ignored @IncludeOnly(mmp.supported_extensions) @@ -94,7 +101,6 @@ class StoreWatchListener(BaseListener, Loggable, pyinotify.ProcessEvent): @mediate_ignored @IncludeOnly(mmp.supported_extensions) def process_delete(self, event): - print(event) dispatcher.send(signal=self.signal, sender=self, event=DeleteFile(event)) def flush_events(self, path): diff --git a/python_apps/media-monitor2/media/monitor/pure.py b/python_apps/media-monitor2/media/monitor/pure.py index b705ddc9b..29f1ef366 100644 --- a/python_apps/media-monitor2/media/monitor/pure.py +++ b/python_apps/media-monitor2/media/monitor/pure.py @@ -7,7 +7,7 @@ import hashlib from configobj import ConfigObj import locale -from media.monitor.exceptions import FailedToSetLocale +from media.monitor.exceptions import FailedToSetLocale, FailedToCreateDir supported_extensions = [u"mp3", u"ogg"] unicode_unknown = u'unknown' @@ -308,6 +308,17 @@ def import_organize(store): store = os.path.normpath(store) return os.path.join(store,'organize'), os.path.join(store,'imported') +def create_dir(path): + """ + will try and make sure that path exists at all costs. raises an exception + if it fails at this task. + """ + if not os.path.exists(path): + try: os.makedirs(path) + except Exception as e: raise FailedToCreateDir(path, e) + else: # if no error occurs we still need to check that dir exists + if not os.path.exists: raise FailedToCreateDir(path) + if __name__ == '__main__': import doctest doctest.testmod() diff --git a/python_apps/media-monitor2/media/monitor/watchersyncer.py b/python_apps/media-monitor2/media/monitor/watchersyncer.py index 538b301ad..0ca1da4bc 100644 --- a/python_apps/media-monitor2/media/monitor/watchersyncer.py +++ b/python_apps/media-monitor2/media/monitor/watchersyncer.py @@ -74,7 +74,7 @@ class TimeoutWatcher(threading.Thread,Loggable): self.watcher.flush_events() class WatchSyncer(ReportHandler,Loggable): - def __init__(self, signal, chunking_number = 50, timeout=15): + def __init__(self, signal, chunking_number = 100, timeout=15): self.path = '' # TODO : get rid of this attribute everywhere #self.signal = signal self.timeout = timeout diff --git a/python_apps/media-monitor2/mm2.py b/python_apps/media-monitor2/mm2.py index 8a7e5e937..997374637 100644 --- a/python_apps/media-monitor2/mm2.py +++ b/python_apps/media-monitor2/mm2.py @@ -5,12 +5,14 @@ import os from media.monitor.manager import Manager from media.monitor.bootstrap import Bootstrapper from media.monitor.log import get_logger +from media.monitor.events import PathChannel from media.monitor.config import MMConfig from media.monitor.toucher import ToucherThread from media.monitor.syncdb import SyncDB -from media.monitor.exceptions import FailedToObtainLocale, FailedToSetLocale, NoConfigFile +from media.monitor.exceptions import FailedToObtainLocale, FailedToSetLocale, NoConfigFile, FailedToCreateDir from media.monitor.airtime import AirtimeNotifier, AirtimeMessageReceiver from media.monitor.watchersyncer import WatchSyncer +from media.monitor.handler import ProblemFileHandler from media.monitor.eventdrainer import EventDrainer import media.monitor.pure as mmp @@ -60,6 +62,10 @@ except Exception as e: log.info(str(e)) watch_syncer = WatchSyncer(signal='watch') +try: + problem_handler = ProblemFileHandler( PathChannel(signal='badfile',path='/srv/airtime/stor/problem_files/') ) +except FailedToCreateDir as e: + log.info("Failed to create problem directory: '%s'" % e.path) apiclient = apc.AirtimeApiClient.create_right_config(log=log,config_path=global_config)