From be00cc69909535c930cf75f15e128035943a550a Mon Sep 17 00:00:00 2001
From: Rudi Grinberg <rudi.grinberg@sourcefabric.org>
Date: Thu, 26 Jul 2012 15:49:41 -0400
Subject: [PATCH] cc-4105: major refactorings. shit is barely running

---
 python_apps/api_clients/api_client.py         |  13 ++
 .../media-monitor2/media/monitor/airtime.py   |  83 ++++++++--
 .../media-monitor2/media/monitor/bootstrap.py |  81 +++++-----
 .../media/monitor/exceptions.py               |   4 +
 .../media-monitor2/media/monitor/legacy.py    | 149 ------------------
 .../media-monitor2/media/monitor/listeners.py |   2 +-
 .../media-monitor2/media/monitor/manager.py   |  10 +-
 .../media-monitor2/media/monitor/pure.py      |   5 +
 .../media/monitor/watchersyncer.py            |  15 +-
 python_apps/media-monitor2/mm2.py             |  92 +++--------
 python_apps/media-monitor2/tests/run_tests.sh |   6 -
 .../media-monitor2/tests/test_api_client.py   |   5 +-
 .../media-monitor2/tests/test_notifier.py     |   4 +-
 .../media-monitor2/tests/test_syncdb.py       |   6 +-
 14 files changed, 185 insertions(+), 290 deletions(-)
 delete mode 100644 python_apps/media-monitor2/media/monitor/legacy.py
 delete mode 100755 python_apps/media-monitor2/tests/run_tests.sh

diff --git a/python_apps/api_clients/api_client.py b/python_apps/api_clients/api_client.py
index efe079a35..bb5fbe737 100644
--- a/python_apps/api_clients/api_client.py
+++ b/python_apps/api_clients/api_client.py
@@ -41,6 +41,19 @@ def convert_dict_value_to_utf8(md):
 
 class AirtimeApiClient():
 
+    # This is a little hacky fix so that I don't have to pass the config object
+    # everywhere where AirtimeApiClient needs to be initialized
+    default_config = None
+    # the purpose of this custom constructor is to remember which config file
+    # it was called with. So that after the initial call:
+    # AirtimeApiClient.create_right_config('/path/to/config')
+    # All subsequence calls to create_right_config will be with that config
+    # file
+    @staticmethod
+    def create_right_config(log=None,config_path=None):
+        if config_path: default_config = config_path
+        return AirtimeApiClient( logger=None, config_path=default_config )
+
     def __init__(self, logger=None,config_path='/etc/airtime/api_client.cfg'):
         if logger is None:
             self.logger = logging
diff --git a/python_apps/media-monitor2/media/monitor/airtime.py b/python_apps/media-monitor2/media/monitor/airtime.py
index ffae74a92..cf1ff7f01 100644
--- a/python_apps/media-monitor2/media/monitor/airtime.py
+++ b/python_apps/media-monitor2/media/monitor/airtime.py
@@ -3,11 +3,17 @@ from kombu.messaging import Exchange, Queue, Consumer
 from kombu.connection import BrokerConnection
 import json
 import os
+import time
 import copy
 
 from media.monitor.exceptions import BadSongFile
 from media.monitor.metadata import Metadata
 from media.monitor.log import Loggable
+from media.monitor.syncdb import SyncDB
+from media.monitor.exceptions import DirectoryIsNotListed
+from media.monitor.bootstrap import Bootstrapper
+
+from api_clients import api_client as apc
 
 # Do not confuse with media monitor 1's AirtimeNotifier class that more related
 # to pyinotify's Notifier class. AirtimeNotifier just notifies when events come
@@ -54,7 +60,7 @@ class AirtimeNotifier(Loggable):
 
 
 class AirtimeMessageReceiver(Loggable):
-    def __init__(self, cfg):
+    def __init__(self, cfg, manager):
         self.dispatch_table = {
                 'md_update' : self.md_update,
                 'new_watch' : self.new_watch,
@@ -64,6 +70,7 @@ class AirtimeMessageReceiver(Loggable):
                 'file_delete' : self.file_delete,
         }
         self.cfg = cfg
+        self.manager = manager
     def message(self, msg):
         """
         This method is called by an AirtimeNotifier instance that consumes the Rabbit MQ events
@@ -84,13 +91,22 @@ class AirtimeMessageReceiver(Loggable):
     def _execute_message(self,evt,message):
         self.dispatch_table[evt](message)
 
+
+    def __request_now_bootstrap(self, directory_id=None, directory=None):
+        sdb = SyncDB(apc.AirtimeApiClient.create_right_config())
+        if directory_id == None: directory_id = sdb.directories[directory]
+        if directory_id in sdb.id_lookup:
+            d = sdb.id_lookup[directory_id]
+            bs = Bootstrapper(sdb, self.manager.watch_signal())
+            bs.flush_watch( directory=d, last_ran=time.time() )
+        else:
+            raise DirectoryIsNotListed(directory_id)
+
     def supported_messages(self):
         return self.dispatch_table.keys()
 
-    # TODO : Handler methods - Should either fire the events directly with
-    # pydispatcher or do the necessary changes on the filesystem that will fire
-    # the events
     def md_update(self, msg):
+        self.logger.info("Updating metadata for: '%s'" % msg['MDATA_KEY_FILEPATH'])
         md_path = msg['MDATA_KEY_FILEPATH']
         try:
             Metadata.write_unsafe(path=md_path, md=msg)
@@ -101,21 +117,65 @@ class AirtimeMessageReceiver(Loggable):
             self.logger.info("Unknown error when writing metadata to: '%s'" % md_path)
 
     def new_watch(self, msg):
-        # TODO: "walk" the newly watched directory
-        self.manager.add_watch_directory(msg['directory'])
+        self.logger.info("Creating watch for directory: '%s'" % msg['directory'])
+        if not os.path.exists(msg['directory']):
+            try: os.makedirs(msg['directory'])
+            except Exception as e:
+                self.logger.info("Failed to create watched dir '%s'" % msg['directory'])
+                self.logger.info(str(e))
+            # Is this clever or stupid?
+            else: self.new_watch(msg)
+        else:
+            # TODO : Refactor this; breaks encapsulation.
+            self.manager.watch_listener.flush_events(msg['directory'])
+            self.manager.add_watch_directory(msg['directory'])
+
     def remove_watch(self, msg):
+        self.logger.info("Removing watch from directory: '%s'" % msg['directory'])
         self.manager.remove_watch_directory(msg['directory'])
+
     def rescan_watch(self, msg):
-        # TODO : basically a bootstrap on the directory
-        pass
+        self.logger.info("Trying to rescan watched directory: '%s'" % msg['directory'])
+        try:
+            self.__request_now_bootstrap(msg['id'])
+        except DirectoryIsNotListed as e:
+            self.logger.info("Bad rescan request")
+            self.logger.info( str(e) )
+        except Exception as e:
+            self.logger.info("Bad rescan request. Unknown error.")
+            self.logger.info( str(e) )
+        else:
+            self.logger.info("Successfully re-scanned: '%s'" % msg['directory'])
+
     def change_storage(self, msg):
         new_storage_directory = msg['directory']
-        new_storage_directory_id = str(msg['dir_id'])
+        new_import = os.path.join(new_storage_directory, 'imported')
+        new_organize = os.path.join(new_storage_directory, 'organize')
+        for d in [new_import, new_organize]:
+            if os.path.exists(d):
+                self.logger.info("Changing storage to existing dir: '%s'" % d)
+            else:
+                try: os.makedirs(d)
+                except Exception as e:
+                    self.logger.info("Could not create dir for storage '%s'" % d)
+                    self.logger.info(str(e))
+
+        if all([ os.path.exists(d) for d in [new_import, new_organize] ]):
+            self.manager.set_store_path(new_import)
+            try:
+                self.__request_now_bootstrap( directory=new_import )
+            except Exception as e:
+                self.logger.info("Did not bootstrap off directory '%s'. Probably not in airtime db" % new_import)
+                self.logger.info(str(e))
+            # set_organize_path should automatically flush new_organize
+            self.manager.set_organize_path(new_organize)
+        else:
+            self.logger.info("Change storage procedure failed, could not create directories")
 
     def file_delete(self, msg):
         # deletes should be requested only from imported folder but we don't
         # verify that.
-        # clippy confirmation: "are you really sure?"
+        self.logger.info("Attempting to delete(maybe) '%s'" % msg['filepath'])
         if msg['delete']:
             self.logger.info("Clippy confirmation was received, actually deleting file...")
             if os.path.exists(msg['filepath']):
@@ -126,7 +186,8 @@ class AirtimeMessageReceiver(Loggable):
                     self.logger.info("Failed to delete '%s'" % msg['filepath'])
                     self.logger.info("Error: " % str(e))
             else:
-                self.logger.info("Attempting to delete file '%s' that does not exist. Full request coming:" % msg['filepath'])
+                self.logger.info("Attempting to delete file '%s' that does not exist. Full request coming:"
+                        % msg['filepath'])
                 self.logger.info(msg)
         else:
             self.logger.info("No clippy confirmation, ignoring event. Out of curiousity we will print some details.")
diff --git a/python_apps/media-monitor2/media/monitor/bootstrap.py b/python_apps/media-monitor2/media/monitor/bootstrap.py
index 417513638..5baff7cef 100644
--- a/python_apps/media-monitor2/media/monitor/bootstrap.py
+++ b/python_apps/media-monitor2/media/monitor/bootstrap.py
@@ -9,55 +9,48 @@ class Bootstrapper(Loggable):
     Bootstrapper reads all the info in the filesystem flushes organize
     events and watch events
     """
-    def __init__(self,db,last_ran,org_channels,watch_channels):
+    def __init__(self,db,watch_signal):
+        """
+        db - SyncDB object; small layer over api client
+        last_ran - last time the program was ran.
+        watch_signal - the signals should send events for every file on.
+        """
         self.db = db
-        self.org_channels = org_channels
-        self.watch_channels = watch_channels
-        self.last_ran = last_ran
+        self.watch_signal = watch_signal
 
-    def flush_watch(self):
+    def flush_all(self, last_ran):
         """
-        Syncs the file system into the database. Walks over deleted/new/modified files since
-        the last run in mediamonitor and sends requests to make the database consistent with
-        file system
+        bootstrap every single watched directory. only useful at startup
         """
-        # Songs is a dictionary where every key is the watched the directory
-        # and the value is a set with all the files in that directory.
-        songs = {}
+        for d in self.db.list_directories():
+            self.flush_watch(d, last_ran)
+
+    def flush_watch(self, directory, last_ran):
+        """
+        flush a single watch/imported directory. useful when wanting to to rescan,
+        or add a watched/imported directory
+        """
+        songs = set([])
         modded = deleted = 0
-        signal_by_path = dict( (pc.signal, pc.path) for pc in self.watch_channels )
-        for pc in self.watch_channels:
-            songs[ pc.path ] = set()
-            for f in mmp.walk_supported(pc.path, clean_empties=False):
-                songs[ pc.path ].add(f)
-                # We decide whether to update a file's metadata by checking
-                # its system modification date. If it's above the value
-                # self.last_ran which is passed to us that means media monitor
-                # wasn't aware when this changes occured in the filesystem
-                # hence it will send the correct events to sync the database
-                # with the filesystem
-                if os.path.getmtime(f) > self.last_ran:
-                    modded += 1
-                    dispatcher.send(signal=pc.signal, sender=self, event=DeleteFile(f))
-                    dispatcher.send(signal=pc.signal, sender=self, event=NewFile(f))
-        # Want all files in the database that are not in the filesystem
-        for watch_dir in self.db.list_directories():
-            db_songs = self.db.directory_get_files(watch_dir)
-            # Get all the files that are in the database but in the file
-            # system. These are the files marked for deletions
-            for to_delete in db_songs.difference(songs[watch_dir]):
-                # need the correct watch channel signal to call delete
-                if watch_dir in signal_by_path:
-                    dispatcher.send(signal=signal_by_path[watch_dir], sender=self, event=DeleteFile(f))
-                    # TODO : get rid of this, we should never delete files from
-                    # the FS even if they are deleted in airtime. Instead we
-                    # should put this file on a global ignore list until it's
-                    # re-added or something
-                    # os.remove(to_delete)
-                    deleted += 1
-                else:
-                    self.logger.error("Could not find the signal corresponding to path: '%s'" % watch_dir)
+        for f in mmp.walk_supported(directory, clean_empties=False):
+            songs.add(f)
+            # We decide whether to update a file's metadata by checking
+            # its system modification date. If it's above the value
+            # self.last_ran which is passed to us that means media monitor
+            # wasn't aware when this changes occured in the filesystem
+            # hence it will send the correct events to sync the database
+            # with the filesystem
+            if os.path.getmtime(f) > last_ran:
+                modded += 1
+                dispatcher.send(signal=self.watch_signal, sender=self, event=DeleteFile(f))
+                dispatcher.send(signal=self.watch_signal, sender=self, event=NewFile(f))
+        db_songs = self.db.directory_get_files(directory)
+        # Get all the files that are in the database but in the file
+        # system. These are the files marked for deletions
+        for to_delete in db_songs.difference(songs):
+            dispatcher.send(signal=self.watch_signal, sender=self, event=DeleteFile(f))
+            deleted += 1
         self.logger.info( "Flushed watch directories. (modified, deleted) = (%d, %d)"
-                         % (modded, deleted) )
+                        % (modded, deleted) )
 
 
diff --git a/python_apps/media-monitor2/media/monitor/exceptions.py b/python_apps/media-monitor2/media/monitor/exceptions.py
index 333663948..184d7158a 100644
--- a/python_apps/media-monitor2/media/monitor/exceptions.py
+++ b/python_apps/media-monitor2/media/monitor/exceptions.py
@@ -28,3 +28,7 @@ class CouldNotCreateIndexFile(Exception):
         self.cause = cause
     def __str__(self): return "Failed to create touch file '%s'" % self.path
 
+class DirectoryIsNotListed(Exception):
+    def __init__(self,dir_id):
+        self.dir_id = dir_id
+    def __str__(self): return "%d was not listed as a directory in the database" % self.dir_id
diff --git a/python_apps/media-monitor2/media/monitor/legacy.py b/python_apps/media-monitor2/media/monitor/legacy.py
deleted file mode 100644
index 1be1532ae..000000000
--- a/python_apps/media-monitor2/media/monitor/legacy.py
+++ /dev/null
@@ -1,149 +0,0 @@
-import os
-import time
-import media.monitor.pure as mmp
-import media.monitor.log
-from subprocess import Popen, PIPE
-import api_clients.api_client as ac
-from media.monitor.syncdb import SyncDB
-
-logger = media.monitor.log.get_logger()
-
-def find_command(directory, extra_arguments=""):
-    """ Builds a find command that respects supported_file_formats list
-    Note: Use single quotes to quote arguments """
-    ext_globs = [ "-iname '*.%s'" % ext for ext in mmp.supported_extensions ]
-    find_glob = ' -o '.join(ext_globs)
-    return "find '%s' %s %s" % (directory, find_glob, extra_arguments)
-
-def exec_command(command):
-    p = Popen(command, shell=True, stdout=PIPE, stderr=PIPE)
-    stdout, stderr = p.communicate()
-    if p.returncode != 0:
-        logger.warn("command \n%s\n return with a non-zero return value", command)
-        logger.error(stderr)
-    try:
-        #File name charset encoding is UTF-8.
-        stdout = stdout.decode("UTF-8")
-    except Exception:
-        stdout = None
-        logger.error("Could not decode %s using UTF-8" % stdout)
-    return stdout
-
-def scan_dir_for_new_files(dir):
-    command = find_command(directory=dir, extra_arguments="-type f -readable")
-    logger.debug(command)
-    stdout = exec_command(command)
-    if stdout is None: return []
-    else: return stdout.splitlines()
-
-def clean_dirty_file_paths(dirty_files):
-    """ clean dirty file paths by removing blanks and removing trailing/leading whitespace"""
-    return filter(lambda e: len(e) > 0, [ f.strip(" \n") for f in dirty_files ])
-
-def handle_created_file(dir, pathname, name):
-    if not dir:
-        self.logger.debug("PROCESS_IN_CLOSE_WRITE: %s, name: %s, pathname: %s ", dir, name, pathname)
-
-        if self.mmc.is_temp_file(name) :
-            #file created is a tmp file which will be modified and then moved back to the original filename.
-            #Easy Tag creates this when changing metadata of ogg files.
-            self.temp_files[pathname] = None
-        #file is being overwritten/replaced in GUI.
-        elif "goutputstream" in pathname:
-            self.temp_files[pathname] = None
-        elif self.mmc.is_audio_file(name):
-            if self.mmc.is_parent_directory(pathname, self.config.organize_directory):
-
-                #file was created in /srv/airtime/stor/organize. Need to process and move
-                #to /srv/airtime/stor/imported
-                file_md = self.md_manager.get_md_from_file(pathname)
-                playable = self.mmc.test_file_playability(pathname)
-
-                if file_md and playable:
-                    self.mmc.organize_new_file(pathname, file_md)
-                else:
-                    #move to problem_files
-                    self.mmc.move_to_problem_dir(pathname)
-            else:
-                # only append to self.file_events if the file isn't going to be altered by organize_new_file(). If file is going
-                # to be altered by organize_new_file(), then process_IN_MOVED_TO event will handle appending it to self.file_events
-                is_recorded = self.mmc.is_parent_directory(pathname, self.config.recorded_directory)
-                self.file_events.append({'mode': self.config.MODE_CREATE, 'filepath': pathname, 'is_recorded_show': is_recorded})
-def handle_removed_file(dir, pathname):
-    logger.info("Deleting %s", pathname)
-    if not dir:
-        if mmc.is_audio_file(pathname):
-            if pathname in self.ignore_event:
-                logger.info("pathname in ignore event")
-                ignore_event.remove(pathname)
-            elif not self.mmc.is_parent_directory(pathname, self.config.organize_directory):
-                logger.info("deleting a file not in organize")
-                #we don't care if a file was deleted from the organize directory.
-                file_events.append({'filepath': pathname, 'mode': self.config.MODE_DELETE})
-
-"""
-This function takes in a path name provided by the database (and its corresponding row id)
-and reads the list of files in the local file system. Its purpose is to discover which files
-exist on the file system but not in the database and vice versa, as well as which files have
-been modified since the database was last updated. In each case, this method will call an
-appropiate method to ensure that the database actually represents the filesystem.
-dir_id -- row id of the directory in the cc_watched_dirs database table
-dir    -- pathname of the directory
-"""
-def sync_database_to_filesystem(dir_id, dir,syncdb, last_ran=0):
-    # TODO: is this line even necessary?
-    dir = os.path.normpath(dir)+"/"
-    """
-    set to hold new and/or modified files. We use a set to make it ok if files are added
-    twice. This is because some of the tests for new files return result sets that are not
-    mutually exclusive from each other.
-    """
-    removed_files = set() # Not used in the original code either
-    db_known_files_set = set()
-    files = syncdb.id_get_files(dir_id)
-    for f in files:
-        db_known_files_set.add(f)
-    all_files = clean_dirty_file_paths( scan_dir_for_new_files(dir) )
-    all_files_set = set()
-    for file_path in all_files:
-        all_files_set.add(file_path[len(dir):])
-    if last_ran > 0:
-        """find files that have been modified since the last time media-monitor process started."""
-        time_diff_sec = time.time() - last_ran
-        command = find_command(directory=dir, extra_arguments=("-type f -readable -mmin -%d" % (time_diff_sec/60+1)))
-    else:
-        command = find_command(directory=dir, extra_arguments="-type f -readable")
-    logger.debug(command)
-    stdout = exec_command(command)
-    if stdout is None: new_files = []
-    else: new_files = stdout.splitlines()
-    new_and_modified_files = set()
-    for file_path in new_files:
-        new_and_modified_files.add(file_path[len(dir):])
-    """
-    new_and_modified_files gives us a set of files that were either copied or modified
-    since the last time media-monitor was running. These files were collected based on
-    their modified timestamp. But this is not all that has changed in the directory. Files
-    could have been removed, or files could have been moved into this directory (moving does
-    not affect last modified timestamp). Lets get a list of files that are on the file-system
-    that the db has no record of, and vice-versa.
-    """
-    deleted_files_set = db_known_files_set - all_files_set
-    new_files_set = all_files_set - db_known_files_set
-    modified_files_set = new_and_modified_files - new_files_set
-    logger.info(u"Deleted files: \n%s\n\n", deleted_files_set)
-    logger.info(u"New files: \n%s\n\n", new_files_set)
-    logger.info(u"Modified files: \n%s\n\n", modified_files_set)
-    for file_path in deleted_files_set:
-        logger.debug("deleted file")
-        full_file_path = os.path.join(dir, file_path)
-        logger.debug(full_file_path)
-        self.pe.handle_removed_file(False, full_file_path)
-    for file_set, debug_message, handle_attribute in [(new_files_set, "new file", "handle_created_file"),
-                                                        (modified_files_set, "modified file", "handle_modified_file")]:
-        for file_path in file_set:
-            logger.debug(debug_message)
-            full_file_path = os.path.join(dir, file_path)
-            logger.debug(full_file_path)
-            if os.path.exists(full_file_path):
-                getattr(self.pe,handle_attribute)(False,full_file_path, os.path.basename(full_file_path))
diff --git a/python_apps/media-monitor2/media/monitor/listeners.py b/python_apps/media-monitor2/media/monitor/listeners.py
index 5d8db2bb7..767ee2917 100644
--- a/python_apps/media-monitor2/media/monitor/listeners.py
+++ b/python_apps/media-monitor2/media/monitor/listeners.py
@@ -84,6 +84,6 @@ class StoreWatchListener(BaseListener, Loggable, pyinotify.ProcessEvent):
         added = 0
         for f in mmp.walk_supported(path, clean_empties=False):
             added += 1
-            dispatcher.send(signal=self.signal, sender=self, event=NewFile(f))
+            dispatcher.send( signal=self.signal, sender=self, event=NewFile(f) )
         self.logger.info( "Flushed watch directory. added = %d" % added )
 
diff --git a/python_apps/media-monitor2/media/monitor/manager.py b/python_apps/media-monitor2/media/monitor/manager.py
index 3b16e7da9..ecef7332d 100644
--- a/python_apps/media-monitor2/media/monitor/manager.py
+++ b/python_apps/media-monitor2/media/monitor/manager.py
@@ -36,6 +36,10 @@ class Manager(Loggable):
         # removed...
         self.watched_directories = set([])
 
+
+    def watch_signal(self):
+        return self.watch_listener.self.signal
+
     def __remove_watch(self,path):
         if path in self.__wd_path: # only delete if dir is actually being watched
             wd = self.__wd_path[path]
@@ -94,7 +98,7 @@ class Manager(Loggable):
         # new files are added to the database. Note that we are not responsible
         # for removing songs in the old store directory from the database
         # we assume that this is already done for us.
-        self.watch_listener.flush_events(new_path)
+        # self.watch_listener.flush_events(new_path)
         self.__add_watch(new_path, self.watch_listener)
 
     store_path = property(get_store_path, set_store_path)
@@ -130,6 +134,10 @@ class Manager(Loggable):
             self.logger.info("'%s' is not being watched, hence cannot be removed"
                              % watch_dir)
 
+    def pyinotify(self):
+        return pyinotify.Notifier(self.wm)
+
+
     def loop(self):
         """
         block until we receive pyinotify events
diff --git a/python_apps/media-monitor2/media/monitor/pure.py b/python_apps/media-monitor2/media/monitor/pure.py
index 960d0c8a4..3395a88bb 100644
--- a/python_apps/media-monitor2/media/monitor/pure.py
+++ b/python_apps/media-monitor2/media/monitor/pure.py
@@ -287,6 +287,11 @@ def last_modified(path):
         return os.path.getmtime(path)
     else: 0
 
+def import_organize(store):
+    """returns a tuple of organize and imported directory from an airtime store directory"""
+    store = os.path.normpath(store)
+    return os.path.join(store,'organize'), os.path.join(store,'imported')
+
 if __name__ == '__main__':
     import doctest
     doctest.testmod()
diff --git a/python_apps/media-monitor2/media/monitor/watchersyncer.py b/python_apps/media-monitor2/media/monitor/watchersyncer.py
index c1ee9421b..45bc7f3b1 100644
--- a/python_apps/media-monitor2/media/monitor/watchersyncer.py
+++ b/python_apps/media-monitor2/media/monitor/watchersyncer.py
@@ -19,7 +19,7 @@ class RequestSync(threading.Thread,Loggable):
 
     @LazyProperty
     def apiclient(self):
-        return ac.AirtimeApiClient()
+        return ac.AirtimeApiClient.create_right_config()
 
     def run(self):
         # TODO : implement proper request sending
@@ -28,7 +28,7 @@ class RequestSync(threading.Thread,Loggable):
         # Not forget to attach the 'is_record' to any requests that are related
         # to recorded shows
         # A simplistic request would like:
-        # self.apiclient.send_media_monitor_requests(requests)
+        self.apiclient.send_media_monitor_requests(self.requests)
         self.watcher.flag_done()
 
 class TimeoutWatcher(threading.Thread,Loggable):
@@ -56,8 +56,9 @@ class TimeoutWatcher(threading.Thread,Loggable):
                 self.watcher.flush_events()
 
 class WatchSyncer(ReportHandler,Loggable):
-    def __init__(self, channel, chunking_number = 50, timeout=15):
-        self.channel = channel
+    def __init__(self, signal, chunking_number = 50, timeout=15):
+        self.path = '' # TODO : get rid of this attribute everywhere
+        #self.signal = signal
         self.timeout = timeout
         self.chunking_number = chunking_number
         self.__queue = []
@@ -70,12 +71,10 @@ class WatchSyncer(ReportHandler,Loggable):
         tc = TimeoutWatcher(self, timeout)
         tc.daemon = True
         tc.start()
-        super(WatchSyncer, self).__init__(signal=channel.signal)
+        super(WatchSyncer, self).__init__(signal=signal)
 
     @property
-    def target_path(self): return self.channel.path
-    @property
-    def signal(self): return self.channel.signal
+    def target_path(self): return self.path
 
     def handle(self, sender, event):
         """We implement this abstract method from ReportHandler"""
diff --git a/python_apps/media-monitor2/mm2.py b/python_apps/media-monitor2/mm2.py
index 7eef83cb0..6ebf2d4bf 100644
--- a/python_apps/media-monitor2/mm2.py
+++ b/python_apps/media-monitor2/mm2.py
@@ -1,13 +1,8 @@
 # -*- coding: utf-8 -*-
-import pyinotify
 import sys
 import os
 
-from media.monitor.listeners import OrganizeListener, StoreWatchListener
-from media.monitor.organizer import Organizer
-from media.monitor.events import PathChannel
-from media.monitor.watchersyncer import WatchSyncer
-from media.monitor.handler import ProblemFileHandler
+from media.monitor.manager import Manager
 from media.monitor.bootstrap import Bootstrapper
 from media.monitor.log import get_logger
 from media.monitor.config import MMConfig
@@ -15,6 +10,7 @@ from media.monitor.toucher import ToucherThread
 from media.monitor.syncdb import SyncDB
 from media.monitor.exceptions import FailedToObtainLocale, FailedToSetLocale, NoConfigFile
 from media.monitor.airtime import AirtimeNotifier, AirtimeMessageReceiver
+from media.monitor.watchersyncer import WatchSyncer
 import media.monitor.pure as mmp
 
 from api_clients import api_client as apc
@@ -34,7 +30,7 @@ from api_clients import api_client as apc
 # Rewrite to use manager.Manager
 
 log = get_logger()
-global_config = u'/path/to/global/config'
+global_config = u'/home/rudi/Airtime/python_apps/media-monitor2/tests/live_client.cfg'
 # MMConfig is a proxy around ConfigObj instances. it does not allow itself
 # users of MMConfig instances to modify any config options directly through the
 # dictionary. Users of this object muse use the correct methods designated for
@@ -62,80 +58,44 @@ except Exception as e:
     log.info("Failed to set the locale for unknown reason. Logging exception.")
     log.info(str(e))
 
-#channels = {
-     #note that org channel still has a 'watch' path because that is the path
-     #it supposed to be moving the organized files to. it doesn't matter where
-     #are all the "to organize" files are coming from
-    #'org' : PathChannel('org', '/home/rudi/throwaway/fucking_around/organize'),
-    #'watch' : [],
-    #'badfile' : PathChannel('badfile', '/home/rudi/throwaway/fucking_around/problem_dir'),
-#}
+watch_syncer = WatchSyncer(signal='watch')
 
-channels = {}
-org = config['org']
-channels['org'] = PathChannel(org['signal'], org['path'])
-channels['watch'] = []
-problem = config['problem']
-channels['badfile'] = PathChannel(problem['signal'], problem['path'])
+apiclient = apc.AirtimeApiClient.create_right_config(log=log,config_path=global_config)
 
-apiclient = apc.AirtimeApiClient(log)
-# We initialize sdb before anything because we must know what our watched
-# directories are.
+
+# TODO : Need to do setup_media_monitor call somewhere around here to get
+# import/organize dirs
 sdb = SyncDB(apiclient)
 
+manager = Manager()
+
+store = apiclient.setup_media_monitor()
+store = store[u'stor']
+
+organize_dir, import_dir  = mmp.import_organize(store)
+# Order matters here:
+manager.set_store_path(import_dir)
+manager.set_organize_path(organize_dir)
+
 for watch_dir in sdb.list_directories():
     if not os.path.exists(watch_dir):
         # Create the watch_directory here
         try: os.makedirs(watch_dir)
         except Exception as e:
             log.error("Could not create watch directory: '%s' (given from the database)." % watch_dir)
-    # We must do another existence check for the watched directory because we
-    # the creation of it could have failed above
     if os.path.exists(watch_dir):
-        channels['watch'].append(PathChannel('watch', watch_dir))
+        manager.add_watch_directory(watch_dir)
 
-# The stor directory is the first directory in the watched directories list
-org = Organizer(channel=channels['org'],target_path=channels['watch'][0].path)
-# TODO : this is wrong
-watches = [ WatchSyncer(channel=pc) for pc in channels['watch'] ]
-problem_files = ProblemFileHandler(channel=channels['badfile'])
+last_ran=config.last_ran()
+bs = Bootstrapper( db=sdb, watch_signal='watch' )
 
-# A slight incosistency here, channels['watch'] is already a list while the
-# other items are single elements. For consistency we should make all the
-# values in channels lists later on
-# we try to not share the config object as much as possible and in this case we
-# prefer to only pass the necessary last_ran parameter instead of the whole
-# object. although this might change in the future if Bootstrapper becomes more
-# complicated
-bs = Bootstrapper(db=sdb, last_ran=config.last_ran(), org_channels=[channels['org']], watch_channels=channels['watch'])
+bs.flush_all( config.last_ran() )
 
-bs.flush_organize()
-bs.flush_watch()
-
-wm = pyinotify.WatchManager()
-
-# Listeners don't care about which directory they're related to. All they care
-# about is which signal they should respond to
-o1 = OrganizeListener(signal=channels['org'].signal)
-# We are assuming that the signals are the same for each watched directory here
-o2 = StoreWatchListener(signal=channels['watch'][0].signal)
-
-notifier = pyinotify.Notifier(wm)
-wdd1 = wm.add_watch(channels['org'].path, pyinotify.ALL_EVENTS, rec=True, auto_add=True, proc_fun=o1)
-for pc in channels['watch']:
-    wdd2 = wm.add_watch(pc.path, pyinotify.ALL_EVENTS, rec=True, auto_add=True, proc_fun=o2)
-
-# After finishing the bootstrapping + the listeners we should initialize the
-# kombu message consumer to respond to messages from airtime. we prefer to
-# leave this component of the program for last because without the *Listener
-# objects we cannot properly respond to all events from airtime anyway.
-
-airtime_receiver = AirtimeMessageReceiver(config)
+airtime_receiver = AirtimeMessageReceiver(config,manager)
 airtime_notifier = AirtimeNotifier(config, airtime_receiver)
-
 # Launch the toucher that updates the last time when the script was ran every
 # n seconds.
-tt = ToucherThread(path=config['index_path'], interval=config['touch_interval'])
-
-notifier.loop()
+tt = ToucherThread(path=config['index_path'], interval=int(config['touch_interval']))
 
+pyi = manager.pyinotify()
+pyi.loop()
diff --git a/python_apps/media-monitor2/tests/run_tests.sh b/python_apps/media-monitor2/tests/run_tests.sh
deleted file mode 100755
index e25911505..000000000
--- a/python_apps/media-monitor2/tests/run_tests.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/bin/bash
-
-for f in /home/rudi/Airtime/python_apps/media-monitor2/tests/*.py
-do
-    python $f
-done
diff --git a/python_apps/media-monitor2/tests/test_api_client.py b/python_apps/media-monitor2/tests/test_api_client.py
index 6601e120d..f89b5efb3 100644
--- a/python_apps/media-monitor2/tests/test_api_client.py
+++ b/python_apps/media-monitor2/tests/test_api_client.py
@@ -4,9 +4,12 @@ import os
 import sys
 from api_clients import api_client as apc
 
+
+import prepare_tests
+
 class TestApiClient(unittest.TestCase):
     def setUp(self):
-        test_path = '/home/rudi/Airtime/python_apps/media-monitor2/tests/api_client.cfg'
+        test_path = prepare_tests.real_config
         if not os.path.exists(test_path):
             print("path for config does not exist: '%s' % test_path")
             # TODO : is there a cleaner way to exit the unit testing?
diff --git a/python_apps/media-monitor2/tests/test_notifier.py b/python_apps/media-monitor2/tests/test_notifier.py
index e56bdc99f..39866dde9 100644
--- a/python_apps/media-monitor2/tests/test_notifier.py
+++ b/python_apps/media-monitor2/tests/test_notifier.py
@@ -6,13 +6,15 @@ from media.monitor.airtime import AirtimeNotifier, AirtimeMessageReceiver
 from mock import patch, Mock
 from media.monitor.config import MMConfig
 
+from media.monitor.manager import Manager
+
 def filter_ev(d): return { i : j for i,j in d.iteritems() if i != 'event_type' }
 
 class TestReceiver(unittest.TestCase):
     def setUp(self):
         # TODO : properly mock this later
         cfg = {}
-        self.amr = AirtimeMessageReceiver(cfg)
+        self.amr = AirtimeMessageReceiver(cfg, Manager())
 
     def test_supported_messages(self):
         self.assertTrue( len(self.amr.supported_messages()) > 0 )
diff --git a/python_apps/media-monitor2/tests/test_syncdb.py b/python_apps/media-monitor2/tests/test_syncdb.py
index 93e083e8b..f3d1fd8ca 100644
--- a/python_apps/media-monitor2/tests/test_syncdb.py
+++ b/python_apps/media-monitor2/tests/test_syncdb.py
@@ -5,10 +5,12 @@ from media.monitor.syncdb import SyncDB
 from media.monitor.log import get_logger
 from media.monitor.pure import partition
 import api_clients.api_client as ac
+import prepare_tests
 
 class TestSyncDB(unittest.TestCase):
     def setUp(self):
-        self.ac = ac.AirtimeApiClient(logger=get_logger())
+        self.ac = ac.AirtimeApiClient(logger=get_logger(),
+                                      config_path=prepare_tests.real_config)
 
     def test_syncdb_init(self):
         sdb = SyncDB(self.ac)
@@ -25,7 +27,7 @@ class TestSyncDB(unittest.TestCase):
         for wdir in sdb.list_directories():
             files = sdb.directory_get_files(wdir)
             self.assertTrue( len(files) >= 0 )
-            self.assertTrue( isinstance(files, list) )
+            self.assertTrue( isinstance(files, set) )
             exist, deleted = partition(os.path.exists, files)
             print("(exist, deleted) = (%d, %d)" % ( len(exist), len(deleted) ) )