From be00cc69909535c930cf75f15e128035943a550a Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 26 Jul 2012 15:49:41 -0400 Subject: [PATCH] cc-4105: major refactorings. shit is barely running --- python_apps/api_clients/api_client.py | 13 ++ .../media-monitor2/media/monitor/airtime.py | 83 ++++++++-- .../media-monitor2/media/monitor/bootstrap.py | 81 +++++----- .../media/monitor/exceptions.py | 4 + .../media-monitor2/media/monitor/legacy.py | 149 ------------------ .../media-monitor2/media/monitor/listeners.py | 2 +- .../media-monitor2/media/monitor/manager.py | 10 +- .../media-monitor2/media/monitor/pure.py | 5 + .../media/monitor/watchersyncer.py | 15 +- python_apps/media-monitor2/mm2.py | 92 +++-------- python_apps/media-monitor2/tests/run_tests.sh | 6 - .../media-monitor2/tests/test_api_client.py | 5 +- .../media-monitor2/tests/test_notifier.py | 4 +- .../media-monitor2/tests/test_syncdb.py | 6 +- 14 files changed, 185 insertions(+), 290 deletions(-) delete mode 100644 python_apps/media-monitor2/media/monitor/legacy.py delete mode 100755 python_apps/media-monitor2/tests/run_tests.sh diff --git a/python_apps/api_clients/api_client.py b/python_apps/api_clients/api_client.py index efe079a35..bb5fbe737 100644 --- a/python_apps/api_clients/api_client.py +++ b/python_apps/api_clients/api_client.py @@ -41,6 +41,19 @@ def convert_dict_value_to_utf8(md): class AirtimeApiClient(): + # This is a little hacky fix so that I don't have to pass the config object + # everywhere where AirtimeApiClient needs to be initialized + default_config = None + # the purpose of this custom constructor is to remember which config file + # it was called with. So that after the initial call: + # AirtimeApiClient.create_right_config('/path/to/config') + # All subsequence calls to create_right_config will be with that config + # file + @staticmethod + def create_right_config(log=None,config_path=None): + if config_path: default_config = config_path + return AirtimeApiClient( logger=None, config_path=default_config ) + def __init__(self, logger=None,config_path='/etc/airtime/api_client.cfg'): if logger is None: self.logger = logging diff --git a/python_apps/media-monitor2/media/monitor/airtime.py b/python_apps/media-monitor2/media/monitor/airtime.py index ffae74a92..cf1ff7f01 100644 --- a/python_apps/media-monitor2/media/monitor/airtime.py +++ b/python_apps/media-monitor2/media/monitor/airtime.py @@ -3,11 +3,17 @@ from kombu.messaging import Exchange, Queue, Consumer from kombu.connection import BrokerConnection import json import os +import time import copy from media.monitor.exceptions import BadSongFile from media.monitor.metadata import Metadata from media.monitor.log import Loggable +from media.monitor.syncdb import SyncDB +from media.monitor.exceptions import DirectoryIsNotListed +from media.monitor.bootstrap import Bootstrapper + +from api_clients import api_client as apc # Do not confuse with media monitor 1's AirtimeNotifier class that more related # to pyinotify's Notifier class. AirtimeNotifier just notifies when events come @@ -54,7 +60,7 @@ class AirtimeNotifier(Loggable): class AirtimeMessageReceiver(Loggable): - def __init__(self, cfg): + def __init__(self, cfg, manager): self.dispatch_table = { 'md_update' : self.md_update, 'new_watch' : self.new_watch, @@ -64,6 +70,7 @@ class AirtimeMessageReceiver(Loggable): 'file_delete' : self.file_delete, } self.cfg = cfg + self.manager = manager def message(self, msg): """ This method is called by an AirtimeNotifier instance that consumes the Rabbit MQ events @@ -84,13 +91,22 @@ class AirtimeMessageReceiver(Loggable): def _execute_message(self,evt,message): self.dispatch_table[evt](message) + + def __request_now_bootstrap(self, directory_id=None, directory=None): + sdb = SyncDB(apc.AirtimeApiClient.create_right_config()) + if directory_id == None: directory_id = sdb.directories[directory] + if directory_id in sdb.id_lookup: + d = sdb.id_lookup[directory_id] + bs = Bootstrapper(sdb, self.manager.watch_signal()) + bs.flush_watch( directory=d, last_ran=time.time() ) + else: + raise DirectoryIsNotListed(directory_id) + def supported_messages(self): return self.dispatch_table.keys() - # TODO : Handler methods - Should either fire the events directly with - # pydispatcher or do the necessary changes on the filesystem that will fire - # the events def md_update(self, msg): + self.logger.info("Updating metadata for: '%s'" % msg['MDATA_KEY_FILEPATH']) md_path = msg['MDATA_KEY_FILEPATH'] try: Metadata.write_unsafe(path=md_path, md=msg) @@ -101,21 +117,65 @@ class AirtimeMessageReceiver(Loggable): self.logger.info("Unknown error when writing metadata to: '%s'" % md_path) def new_watch(self, msg): - # TODO: "walk" the newly watched directory - self.manager.add_watch_directory(msg['directory']) + self.logger.info("Creating watch for directory: '%s'" % msg['directory']) + if not os.path.exists(msg['directory']): + try: os.makedirs(msg['directory']) + except Exception as e: + self.logger.info("Failed to create watched dir '%s'" % msg['directory']) + self.logger.info(str(e)) + # Is this clever or stupid? + else: self.new_watch(msg) + else: + # TODO : Refactor this; breaks encapsulation. + self.manager.watch_listener.flush_events(msg['directory']) + self.manager.add_watch_directory(msg['directory']) + def remove_watch(self, msg): + self.logger.info("Removing watch from directory: '%s'" % msg['directory']) self.manager.remove_watch_directory(msg['directory']) + def rescan_watch(self, msg): - # TODO : basically a bootstrap on the directory - pass + self.logger.info("Trying to rescan watched directory: '%s'" % msg['directory']) + try: + self.__request_now_bootstrap(msg['id']) + except DirectoryIsNotListed as e: + self.logger.info("Bad rescan request") + self.logger.info( str(e) ) + except Exception as e: + self.logger.info("Bad rescan request. Unknown error.") + self.logger.info( str(e) ) + else: + self.logger.info("Successfully re-scanned: '%s'" % msg['directory']) + def change_storage(self, msg): new_storage_directory = msg['directory'] - new_storage_directory_id = str(msg['dir_id']) + new_import = os.path.join(new_storage_directory, 'imported') + new_organize = os.path.join(new_storage_directory, 'organize') + for d in [new_import, new_organize]: + if os.path.exists(d): + self.logger.info("Changing storage to existing dir: '%s'" % d) + else: + try: os.makedirs(d) + except Exception as e: + self.logger.info("Could not create dir for storage '%s'" % d) + self.logger.info(str(e)) + + if all([ os.path.exists(d) for d in [new_import, new_organize] ]): + self.manager.set_store_path(new_import) + try: + self.__request_now_bootstrap( directory=new_import ) + except Exception as e: + self.logger.info("Did not bootstrap off directory '%s'. Probably not in airtime db" % new_import) + self.logger.info(str(e)) + # set_organize_path should automatically flush new_organize + self.manager.set_organize_path(new_organize) + else: + self.logger.info("Change storage procedure failed, could not create directories") def file_delete(self, msg): # deletes should be requested only from imported folder but we don't # verify that. - # clippy confirmation: "are you really sure?" + self.logger.info("Attempting to delete(maybe) '%s'" % msg['filepath']) if msg['delete']: self.logger.info("Clippy confirmation was received, actually deleting file...") if os.path.exists(msg['filepath']): @@ -126,7 +186,8 @@ class AirtimeMessageReceiver(Loggable): self.logger.info("Failed to delete '%s'" % msg['filepath']) self.logger.info("Error: " % str(e)) else: - self.logger.info("Attempting to delete file '%s' that does not exist. Full request coming:" % msg['filepath']) + self.logger.info("Attempting to delete file '%s' that does not exist. Full request coming:" + % msg['filepath']) self.logger.info(msg) else: self.logger.info("No clippy confirmation, ignoring event. Out of curiousity we will print some details.") diff --git a/python_apps/media-monitor2/media/monitor/bootstrap.py b/python_apps/media-monitor2/media/monitor/bootstrap.py index 417513638..5baff7cef 100644 --- a/python_apps/media-monitor2/media/monitor/bootstrap.py +++ b/python_apps/media-monitor2/media/monitor/bootstrap.py @@ -9,55 +9,48 @@ class Bootstrapper(Loggable): Bootstrapper reads all the info in the filesystem flushes organize events and watch events """ - def __init__(self,db,last_ran,org_channels,watch_channels): + def __init__(self,db,watch_signal): + """ + db - SyncDB object; small layer over api client + last_ran - last time the program was ran. + watch_signal - the signals should send events for every file on. + """ self.db = db - self.org_channels = org_channels - self.watch_channels = watch_channels - self.last_ran = last_ran + self.watch_signal = watch_signal - def flush_watch(self): + def flush_all(self, last_ran): """ - Syncs the file system into the database. Walks over deleted/new/modified files since - the last run in mediamonitor and sends requests to make the database consistent with - file system + bootstrap every single watched directory. only useful at startup """ - # Songs is a dictionary where every key is the watched the directory - # and the value is a set with all the files in that directory. - songs = {} + for d in self.db.list_directories(): + self.flush_watch(d, last_ran) + + def flush_watch(self, directory, last_ran): + """ + flush a single watch/imported directory. useful when wanting to to rescan, + or add a watched/imported directory + """ + songs = set([]) modded = deleted = 0 - signal_by_path = dict( (pc.signal, pc.path) for pc in self.watch_channels ) - for pc in self.watch_channels: - songs[ pc.path ] = set() - for f in mmp.walk_supported(pc.path, clean_empties=False): - songs[ pc.path ].add(f) - # We decide whether to update a file's metadata by checking - # its system modification date. If it's above the value - # self.last_ran which is passed to us that means media monitor - # wasn't aware when this changes occured in the filesystem - # hence it will send the correct events to sync the database - # with the filesystem - if os.path.getmtime(f) > self.last_ran: - modded += 1 - dispatcher.send(signal=pc.signal, sender=self, event=DeleteFile(f)) - dispatcher.send(signal=pc.signal, sender=self, event=NewFile(f)) - # Want all files in the database that are not in the filesystem - for watch_dir in self.db.list_directories(): - db_songs = self.db.directory_get_files(watch_dir) - # Get all the files that are in the database but in the file - # system. These are the files marked for deletions - for to_delete in db_songs.difference(songs[watch_dir]): - # need the correct watch channel signal to call delete - if watch_dir in signal_by_path: - dispatcher.send(signal=signal_by_path[watch_dir], sender=self, event=DeleteFile(f)) - # TODO : get rid of this, we should never delete files from - # the FS even if they are deleted in airtime. Instead we - # should put this file on a global ignore list until it's - # re-added or something - # os.remove(to_delete) - deleted += 1 - else: - self.logger.error("Could not find the signal corresponding to path: '%s'" % watch_dir) + for f in mmp.walk_supported(directory, clean_empties=False): + songs.add(f) + # We decide whether to update a file's metadata by checking + # its system modification date. If it's above the value + # self.last_ran which is passed to us that means media monitor + # wasn't aware when this changes occured in the filesystem + # hence it will send the correct events to sync the database + # with the filesystem + if os.path.getmtime(f) > last_ran: + modded += 1 + dispatcher.send(signal=self.watch_signal, sender=self, event=DeleteFile(f)) + dispatcher.send(signal=self.watch_signal, sender=self, event=NewFile(f)) + db_songs = self.db.directory_get_files(directory) + # Get all the files that are in the database but in the file + # system. These are the files marked for deletions + for to_delete in db_songs.difference(songs): + dispatcher.send(signal=self.watch_signal, sender=self, event=DeleteFile(f)) + deleted += 1 self.logger.info( "Flushed watch directories. (modified, deleted) = (%d, %d)" - % (modded, deleted) ) + % (modded, deleted) ) diff --git a/python_apps/media-monitor2/media/monitor/exceptions.py b/python_apps/media-monitor2/media/monitor/exceptions.py index 333663948..184d7158a 100644 --- a/python_apps/media-monitor2/media/monitor/exceptions.py +++ b/python_apps/media-monitor2/media/monitor/exceptions.py @@ -28,3 +28,7 @@ class CouldNotCreateIndexFile(Exception): self.cause = cause def __str__(self): return "Failed to create touch file '%s'" % self.path +class DirectoryIsNotListed(Exception): + def __init__(self,dir_id): + self.dir_id = dir_id + def __str__(self): return "%d was not listed as a directory in the database" % self.dir_id diff --git a/python_apps/media-monitor2/media/monitor/legacy.py b/python_apps/media-monitor2/media/monitor/legacy.py deleted file mode 100644 index 1be1532ae..000000000 --- a/python_apps/media-monitor2/media/monitor/legacy.py +++ /dev/null @@ -1,149 +0,0 @@ -import os -import time -import media.monitor.pure as mmp -import media.monitor.log -from subprocess import Popen, PIPE -import api_clients.api_client as ac -from media.monitor.syncdb import SyncDB - -logger = media.monitor.log.get_logger() - -def find_command(directory, extra_arguments=""): - """ Builds a find command that respects supported_file_formats list - Note: Use single quotes to quote arguments """ - ext_globs = [ "-iname '*.%s'" % ext for ext in mmp.supported_extensions ] - find_glob = ' -o '.join(ext_globs) - return "find '%s' %s %s" % (directory, find_glob, extra_arguments) - -def exec_command(command): - p = Popen(command, shell=True, stdout=PIPE, stderr=PIPE) - stdout, stderr = p.communicate() - if p.returncode != 0: - logger.warn("command \n%s\n return with a non-zero return value", command) - logger.error(stderr) - try: - #File name charset encoding is UTF-8. - stdout = stdout.decode("UTF-8") - except Exception: - stdout = None - logger.error("Could not decode %s using UTF-8" % stdout) - return stdout - -def scan_dir_for_new_files(dir): - command = find_command(directory=dir, extra_arguments="-type f -readable") - logger.debug(command) - stdout = exec_command(command) - if stdout is None: return [] - else: return stdout.splitlines() - -def clean_dirty_file_paths(dirty_files): - """ clean dirty file paths by removing blanks and removing trailing/leading whitespace""" - return filter(lambda e: len(e) > 0, [ f.strip(" \n") for f in dirty_files ]) - -def handle_created_file(dir, pathname, name): - if not dir: - self.logger.debug("PROCESS_IN_CLOSE_WRITE: %s, name: %s, pathname: %s ", dir, name, pathname) - - if self.mmc.is_temp_file(name) : - #file created is a tmp file which will be modified and then moved back to the original filename. - #Easy Tag creates this when changing metadata of ogg files. - self.temp_files[pathname] = None - #file is being overwritten/replaced in GUI. - elif "goutputstream" in pathname: - self.temp_files[pathname] = None - elif self.mmc.is_audio_file(name): - if self.mmc.is_parent_directory(pathname, self.config.organize_directory): - - #file was created in /srv/airtime/stor/organize. Need to process and move - #to /srv/airtime/stor/imported - file_md = self.md_manager.get_md_from_file(pathname) - playable = self.mmc.test_file_playability(pathname) - - if file_md and playable: - self.mmc.organize_new_file(pathname, file_md) - else: - #move to problem_files - self.mmc.move_to_problem_dir(pathname) - else: - # only append to self.file_events if the file isn't going to be altered by organize_new_file(). If file is going - # to be altered by organize_new_file(), then process_IN_MOVED_TO event will handle appending it to self.file_events - is_recorded = self.mmc.is_parent_directory(pathname, self.config.recorded_directory) - self.file_events.append({'mode': self.config.MODE_CREATE, 'filepath': pathname, 'is_recorded_show': is_recorded}) -def handle_removed_file(dir, pathname): - logger.info("Deleting %s", pathname) - if not dir: - if mmc.is_audio_file(pathname): - if pathname in self.ignore_event: - logger.info("pathname in ignore event") - ignore_event.remove(pathname) - elif not self.mmc.is_parent_directory(pathname, self.config.organize_directory): - logger.info("deleting a file not in organize") - #we don't care if a file was deleted from the organize directory. - file_events.append({'filepath': pathname, 'mode': self.config.MODE_DELETE}) - -""" -This function takes in a path name provided by the database (and its corresponding row id) -and reads the list of files in the local file system. Its purpose is to discover which files -exist on the file system but not in the database and vice versa, as well as which files have -been modified since the database was last updated. In each case, this method will call an -appropiate method to ensure that the database actually represents the filesystem. -dir_id -- row id of the directory in the cc_watched_dirs database table -dir -- pathname of the directory -""" -def sync_database_to_filesystem(dir_id, dir,syncdb, last_ran=0): - # TODO: is this line even necessary? - dir = os.path.normpath(dir)+"/" - """ - set to hold new and/or modified files. We use a set to make it ok if files are added - twice. This is because some of the tests for new files return result sets that are not - mutually exclusive from each other. - """ - removed_files = set() # Not used in the original code either - db_known_files_set = set() - files = syncdb.id_get_files(dir_id) - for f in files: - db_known_files_set.add(f) - all_files = clean_dirty_file_paths( scan_dir_for_new_files(dir) ) - all_files_set = set() - for file_path in all_files: - all_files_set.add(file_path[len(dir):]) - if last_ran > 0: - """find files that have been modified since the last time media-monitor process started.""" - time_diff_sec = time.time() - last_ran - command = find_command(directory=dir, extra_arguments=("-type f -readable -mmin -%d" % (time_diff_sec/60+1))) - else: - command = find_command(directory=dir, extra_arguments="-type f -readable") - logger.debug(command) - stdout = exec_command(command) - if stdout is None: new_files = [] - else: new_files = stdout.splitlines() - new_and_modified_files = set() - for file_path in new_files: - new_and_modified_files.add(file_path[len(dir):]) - """ - new_and_modified_files gives us a set of files that were either copied or modified - since the last time media-monitor was running. These files were collected based on - their modified timestamp. But this is not all that has changed in the directory. Files - could have been removed, or files could have been moved into this directory (moving does - not affect last modified timestamp). Lets get a list of files that are on the file-system - that the db has no record of, and vice-versa. - """ - deleted_files_set = db_known_files_set - all_files_set - new_files_set = all_files_set - db_known_files_set - modified_files_set = new_and_modified_files - new_files_set - logger.info(u"Deleted files: \n%s\n\n", deleted_files_set) - logger.info(u"New files: \n%s\n\n", new_files_set) - logger.info(u"Modified files: \n%s\n\n", modified_files_set) - for file_path in deleted_files_set: - logger.debug("deleted file") - full_file_path = os.path.join(dir, file_path) - logger.debug(full_file_path) - self.pe.handle_removed_file(False, full_file_path) - for file_set, debug_message, handle_attribute in [(new_files_set, "new file", "handle_created_file"), - (modified_files_set, "modified file", "handle_modified_file")]: - for file_path in file_set: - logger.debug(debug_message) - full_file_path = os.path.join(dir, file_path) - logger.debug(full_file_path) - if os.path.exists(full_file_path): - getattr(self.pe,handle_attribute)(False,full_file_path, os.path.basename(full_file_path)) diff --git a/python_apps/media-monitor2/media/monitor/listeners.py b/python_apps/media-monitor2/media/monitor/listeners.py index 5d8db2bb7..767ee2917 100644 --- a/python_apps/media-monitor2/media/monitor/listeners.py +++ b/python_apps/media-monitor2/media/monitor/listeners.py @@ -84,6 +84,6 @@ class StoreWatchListener(BaseListener, Loggable, pyinotify.ProcessEvent): added = 0 for f in mmp.walk_supported(path, clean_empties=False): added += 1 - dispatcher.send(signal=self.signal, sender=self, event=NewFile(f)) + dispatcher.send( signal=self.signal, sender=self, event=NewFile(f) ) self.logger.info( "Flushed watch directory. added = %d" % added ) diff --git a/python_apps/media-monitor2/media/monitor/manager.py b/python_apps/media-monitor2/media/monitor/manager.py index 3b16e7da9..ecef7332d 100644 --- a/python_apps/media-monitor2/media/monitor/manager.py +++ b/python_apps/media-monitor2/media/monitor/manager.py @@ -36,6 +36,10 @@ class Manager(Loggable): # removed... self.watched_directories = set([]) + + def watch_signal(self): + return self.watch_listener.self.signal + def __remove_watch(self,path): if path in self.__wd_path: # only delete if dir is actually being watched wd = self.__wd_path[path] @@ -94,7 +98,7 @@ class Manager(Loggable): # new files are added to the database. Note that we are not responsible # for removing songs in the old store directory from the database # we assume that this is already done for us. - self.watch_listener.flush_events(new_path) + # self.watch_listener.flush_events(new_path) self.__add_watch(new_path, self.watch_listener) store_path = property(get_store_path, set_store_path) @@ -130,6 +134,10 @@ class Manager(Loggable): self.logger.info("'%s' is not being watched, hence cannot be removed" % watch_dir) + def pyinotify(self): + return pyinotify.Notifier(self.wm) + + def loop(self): """ block until we receive pyinotify events diff --git a/python_apps/media-monitor2/media/monitor/pure.py b/python_apps/media-monitor2/media/monitor/pure.py index 960d0c8a4..3395a88bb 100644 --- a/python_apps/media-monitor2/media/monitor/pure.py +++ b/python_apps/media-monitor2/media/monitor/pure.py @@ -287,6 +287,11 @@ def last_modified(path): return os.path.getmtime(path) else: 0 +def import_organize(store): + """returns a tuple of organize and imported directory from an airtime store directory""" + store = os.path.normpath(store) + return os.path.join(store,'organize'), os.path.join(store,'imported') + if __name__ == '__main__': import doctest doctest.testmod() diff --git a/python_apps/media-monitor2/media/monitor/watchersyncer.py b/python_apps/media-monitor2/media/monitor/watchersyncer.py index c1ee9421b..45bc7f3b1 100644 --- a/python_apps/media-monitor2/media/monitor/watchersyncer.py +++ b/python_apps/media-monitor2/media/monitor/watchersyncer.py @@ -19,7 +19,7 @@ class RequestSync(threading.Thread,Loggable): @LazyProperty def apiclient(self): - return ac.AirtimeApiClient() + return ac.AirtimeApiClient.create_right_config() def run(self): # TODO : implement proper request sending @@ -28,7 +28,7 @@ class RequestSync(threading.Thread,Loggable): # Not forget to attach the 'is_record' to any requests that are related # to recorded shows # A simplistic request would like: - # self.apiclient.send_media_monitor_requests(requests) + self.apiclient.send_media_monitor_requests(self.requests) self.watcher.flag_done() class TimeoutWatcher(threading.Thread,Loggable): @@ -56,8 +56,9 @@ class TimeoutWatcher(threading.Thread,Loggable): self.watcher.flush_events() class WatchSyncer(ReportHandler,Loggable): - def __init__(self, channel, chunking_number = 50, timeout=15): - self.channel = channel + def __init__(self, signal, chunking_number = 50, timeout=15): + self.path = '' # TODO : get rid of this attribute everywhere + #self.signal = signal self.timeout = timeout self.chunking_number = chunking_number self.__queue = [] @@ -70,12 +71,10 @@ class WatchSyncer(ReportHandler,Loggable): tc = TimeoutWatcher(self, timeout) tc.daemon = True tc.start() - super(WatchSyncer, self).__init__(signal=channel.signal) + super(WatchSyncer, self).__init__(signal=signal) @property - def target_path(self): return self.channel.path - @property - def signal(self): return self.channel.signal + def target_path(self): return self.path def handle(self, sender, event): """We implement this abstract method from ReportHandler""" diff --git a/python_apps/media-monitor2/mm2.py b/python_apps/media-monitor2/mm2.py index 7eef83cb0..6ebf2d4bf 100644 --- a/python_apps/media-monitor2/mm2.py +++ b/python_apps/media-monitor2/mm2.py @@ -1,13 +1,8 @@ # -*- coding: utf-8 -*- -import pyinotify import sys import os -from media.monitor.listeners import OrganizeListener, StoreWatchListener -from media.monitor.organizer import Organizer -from media.monitor.events import PathChannel -from media.monitor.watchersyncer import WatchSyncer -from media.monitor.handler import ProblemFileHandler +from media.monitor.manager import Manager from media.monitor.bootstrap import Bootstrapper from media.monitor.log import get_logger from media.monitor.config import MMConfig @@ -15,6 +10,7 @@ from media.monitor.toucher import ToucherThread from media.monitor.syncdb import SyncDB from media.monitor.exceptions import FailedToObtainLocale, FailedToSetLocale, NoConfigFile from media.monitor.airtime import AirtimeNotifier, AirtimeMessageReceiver +from media.monitor.watchersyncer import WatchSyncer import media.monitor.pure as mmp from api_clients import api_client as apc @@ -34,7 +30,7 @@ from api_clients import api_client as apc # Rewrite to use manager.Manager log = get_logger() -global_config = u'/path/to/global/config' +global_config = u'/home/rudi/Airtime/python_apps/media-monitor2/tests/live_client.cfg' # MMConfig is a proxy around ConfigObj instances. it does not allow itself # users of MMConfig instances to modify any config options directly through the # dictionary. Users of this object muse use the correct methods designated for @@ -62,80 +58,44 @@ except Exception as e: log.info("Failed to set the locale for unknown reason. Logging exception.") log.info(str(e)) -#channels = { - #note that org channel still has a 'watch' path because that is the path - #it supposed to be moving the organized files to. it doesn't matter where - #are all the "to organize" files are coming from - #'org' : PathChannel('org', '/home/rudi/throwaway/fucking_around/organize'), - #'watch' : [], - #'badfile' : PathChannel('badfile', '/home/rudi/throwaway/fucking_around/problem_dir'), -#} +watch_syncer = WatchSyncer(signal='watch') -channels = {} -org = config['org'] -channels['org'] = PathChannel(org['signal'], org['path']) -channels['watch'] = [] -problem = config['problem'] -channels['badfile'] = PathChannel(problem['signal'], problem['path']) +apiclient = apc.AirtimeApiClient.create_right_config(log=log,config_path=global_config) -apiclient = apc.AirtimeApiClient(log) -# We initialize sdb before anything because we must know what our watched -# directories are. + +# TODO : Need to do setup_media_monitor call somewhere around here to get +# import/organize dirs sdb = SyncDB(apiclient) +manager = Manager() + +store = apiclient.setup_media_monitor() +store = store[u'stor'] + +organize_dir, import_dir = mmp.import_organize(store) +# Order matters here: +manager.set_store_path(import_dir) +manager.set_organize_path(organize_dir) + for watch_dir in sdb.list_directories(): if not os.path.exists(watch_dir): # Create the watch_directory here try: os.makedirs(watch_dir) except Exception as e: log.error("Could not create watch directory: '%s' (given from the database)." % watch_dir) - # We must do another existence check for the watched directory because we - # the creation of it could have failed above if os.path.exists(watch_dir): - channels['watch'].append(PathChannel('watch', watch_dir)) + manager.add_watch_directory(watch_dir) -# The stor directory is the first directory in the watched directories list -org = Organizer(channel=channels['org'],target_path=channels['watch'][0].path) -# TODO : this is wrong -watches = [ WatchSyncer(channel=pc) for pc in channels['watch'] ] -problem_files = ProblemFileHandler(channel=channels['badfile']) +last_ran=config.last_ran() +bs = Bootstrapper( db=sdb, watch_signal='watch' ) -# A slight incosistency here, channels['watch'] is already a list while the -# other items are single elements. For consistency we should make all the -# values in channels lists later on -# we try to not share the config object as much as possible and in this case we -# prefer to only pass the necessary last_ran parameter instead of the whole -# object. although this might change in the future if Bootstrapper becomes more -# complicated -bs = Bootstrapper(db=sdb, last_ran=config.last_ran(), org_channels=[channels['org']], watch_channels=channels['watch']) +bs.flush_all( config.last_ran() ) -bs.flush_organize() -bs.flush_watch() - -wm = pyinotify.WatchManager() - -# Listeners don't care about which directory they're related to. All they care -# about is which signal they should respond to -o1 = OrganizeListener(signal=channels['org'].signal) -# We are assuming that the signals are the same for each watched directory here -o2 = StoreWatchListener(signal=channels['watch'][0].signal) - -notifier = pyinotify.Notifier(wm) -wdd1 = wm.add_watch(channels['org'].path, pyinotify.ALL_EVENTS, rec=True, auto_add=True, proc_fun=o1) -for pc in channels['watch']: - wdd2 = wm.add_watch(pc.path, pyinotify.ALL_EVENTS, rec=True, auto_add=True, proc_fun=o2) - -# After finishing the bootstrapping + the listeners we should initialize the -# kombu message consumer to respond to messages from airtime. we prefer to -# leave this component of the program for last because without the *Listener -# objects we cannot properly respond to all events from airtime anyway. - -airtime_receiver = AirtimeMessageReceiver(config) +airtime_receiver = AirtimeMessageReceiver(config,manager) airtime_notifier = AirtimeNotifier(config, airtime_receiver) - # Launch the toucher that updates the last time when the script was ran every # n seconds. -tt = ToucherThread(path=config['index_path'], interval=config['touch_interval']) - -notifier.loop() +tt = ToucherThread(path=config['index_path'], interval=int(config['touch_interval'])) +pyi = manager.pyinotify() +pyi.loop() diff --git a/python_apps/media-monitor2/tests/run_tests.sh b/python_apps/media-monitor2/tests/run_tests.sh deleted file mode 100755 index e25911505..000000000 --- a/python_apps/media-monitor2/tests/run_tests.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -for f in /home/rudi/Airtime/python_apps/media-monitor2/tests/*.py -do - python $f -done diff --git a/python_apps/media-monitor2/tests/test_api_client.py b/python_apps/media-monitor2/tests/test_api_client.py index 6601e120d..f89b5efb3 100644 --- a/python_apps/media-monitor2/tests/test_api_client.py +++ b/python_apps/media-monitor2/tests/test_api_client.py @@ -4,9 +4,12 @@ import os import sys from api_clients import api_client as apc + +import prepare_tests + class TestApiClient(unittest.TestCase): def setUp(self): - test_path = '/home/rudi/Airtime/python_apps/media-monitor2/tests/api_client.cfg' + test_path = prepare_tests.real_config if not os.path.exists(test_path): print("path for config does not exist: '%s' % test_path") # TODO : is there a cleaner way to exit the unit testing? diff --git a/python_apps/media-monitor2/tests/test_notifier.py b/python_apps/media-monitor2/tests/test_notifier.py index e56bdc99f..39866dde9 100644 --- a/python_apps/media-monitor2/tests/test_notifier.py +++ b/python_apps/media-monitor2/tests/test_notifier.py @@ -6,13 +6,15 @@ from media.monitor.airtime import AirtimeNotifier, AirtimeMessageReceiver from mock import patch, Mock from media.monitor.config import MMConfig +from media.monitor.manager import Manager + def filter_ev(d): return { i : j for i,j in d.iteritems() if i != 'event_type' } class TestReceiver(unittest.TestCase): def setUp(self): # TODO : properly mock this later cfg = {} - self.amr = AirtimeMessageReceiver(cfg) + self.amr = AirtimeMessageReceiver(cfg, Manager()) def test_supported_messages(self): self.assertTrue( len(self.amr.supported_messages()) > 0 ) diff --git a/python_apps/media-monitor2/tests/test_syncdb.py b/python_apps/media-monitor2/tests/test_syncdb.py index 93e083e8b..f3d1fd8ca 100644 --- a/python_apps/media-monitor2/tests/test_syncdb.py +++ b/python_apps/media-monitor2/tests/test_syncdb.py @@ -5,10 +5,12 @@ from media.monitor.syncdb import SyncDB from media.monitor.log import get_logger from media.monitor.pure import partition import api_clients.api_client as ac +import prepare_tests class TestSyncDB(unittest.TestCase): def setUp(self): - self.ac = ac.AirtimeApiClient(logger=get_logger()) + self.ac = ac.AirtimeApiClient(logger=get_logger(), + config_path=prepare_tests.real_config) def test_syncdb_init(self): sdb = SyncDB(self.ac) @@ -25,7 +27,7 @@ class TestSyncDB(unittest.TestCase): for wdir in sdb.list_directories(): files = sdb.directory_get_files(wdir) self.assertTrue( len(files) >= 0 ) - self.assertTrue( isinstance(files, list) ) + self.assertTrue( isinstance(files, set) ) exist, deleted = partition(os.path.exists, files) print("(exist, deleted) = (%d, %d)" % ( len(exist), len(deleted) ) )